1 //===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the SelectionDAG::Legalize method.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/Analysis/DebugInfo.h"
15 #include "llvm/CodeGen/Analysis.h"
16 #include "llvm/CodeGen/MachineFunction.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/CodeGen/MachineJumpTableInfo.h"
19 #include "llvm/CodeGen/MachineModuleInfo.h"
20 #include "llvm/CodeGen/PseudoSourceValue.h"
21 #include "llvm/CodeGen/SelectionDAG.h"
22 #include "llvm/Target/TargetFrameLowering.h"
23 #include "llvm/Target/TargetLowering.h"
24 #include "llvm/Target/TargetData.h"
25 #include "llvm/Target/TargetMachine.h"
26 #include "llvm/Target/TargetOptions.h"
27 #include "llvm/CallingConv.h"
28 #include "llvm/Constants.h"
29 #include "llvm/DerivedTypes.h"
30 #include "llvm/Function.h"
31 #include "llvm/GlobalVariable.h"
32 #include "llvm/LLVMContext.h"
33 #include "llvm/Support/CommandLine.h"
34 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/MathExtras.h"
37 #include "llvm/Support/raw_ostream.h"
38 #include "llvm/ADT/DenseMap.h"
39 #include "llvm/ADT/SmallVector.h"
40 #include "llvm/ADT/SmallPtrSet.h"
43 //===----------------------------------------------------------------------===//
44 /// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and
45 /// hacks on it until the target machine can handle it. This involves
46 /// eliminating value sizes the machine cannot handle (promoting small sizes to
47 /// large sizes or splitting up large values into small values) as well as
48 /// eliminating operations the machine cannot handle.
50 /// This code also does a small amount of optimization and recognition of idioms
51 /// as part of its processing. For example, if a target does not support a
52 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
53 /// will attempt merge setcc and brc instructions into brcc's.
56 class SelectionDAGLegalize {
57 const TargetMachine &TM;
58 const TargetLowering &TLI;
60 CodeGenOpt::Level OptLevel;
62 // Libcall insertion helpers.
64 /// LastCALLSEQ_END - This keeps track of the CALLSEQ_END node that has been
65 /// legalized. We use this to ensure that calls are properly serialized
66 /// against each other, including inserted libcalls.
67 SDValue LastCALLSEQ_END;
70 Legal, // The target natively supports this operation.
71 Promote, // This operation should be executed in a larger type.
72 Expand // Try to expand this to other ops, otherwise use a libcall.
75 /// ValueTypeActions - This is a bitvector that contains two bits for each
76 /// value type, where the two bits correspond to the LegalizeAction enum.
77 /// This can be queried with "getTypeAction(VT)".
78 TargetLowering::ValueTypeActionImpl ValueTypeActions;
80 /// LegalizedNodes - For nodes that are of legal width, and that have more
81 /// than one use, this map indicates what regularized operand to use. This
82 /// allows us to avoid legalizing the same thing more than once.
83 DenseMap<SDValue, SDValue> LegalizedNodes;
85 void AddLegalizedOperand(SDValue From, SDValue To) {
86 LegalizedNodes.insert(std::make_pair(From, To));
87 // If someone requests legalization of the new node, return itself.
89 LegalizedNodes.insert(std::make_pair(To, To));
91 // Transfer SDDbgValues.
92 DAG.TransferDbgValues(From, To);
96 SelectionDAGLegalize(SelectionDAG &DAG, CodeGenOpt::Level ol);
98 /// getTypeAction - Return how we should legalize values of this type, either
99 /// it is already legal or we need to expand it into multiple registers of
100 /// smaller integer type, or we need to promote it to a larger type.
101 LegalizeAction getTypeAction(EVT VT) const {
102 return (LegalizeAction)ValueTypeActions.getTypeAction(VT);
105 /// isTypeLegal - Return true if this type is legal on this target.
107 bool isTypeLegal(EVT VT) const {
108 return getTypeAction(VT) == Legal;
114 /// LegalizeOp - We know that the specified value has a legal type.
115 /// Recursively ensure that the operands have legal types, then return the
117 SDValue LegalizeOp(SDValue O);
119 SDValue OptimizeFloatStore(StoreSDNode *ST);
121 /// PerformInsertVectorEltInMemory - Some target cannot handle a variable
122 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
123 /// is necessary to spill the vector being inserted into to memory, perform
124 /// the insert there, and then read the result back.
125 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val,
126 SDValue Idx, DebugLoc dl);
127 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
128 SDValue Idx, DebugLoc dl);
130 /// ShuffleWithNarrowerEltType - Return a vector shuffle operation which
131 /// performs the same shuffe in terms of order or result bytes, but on a type
132 /// whose vector element type is narrower than the original shuffle type.
133 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
134 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, DebugLoc dl,
135 SDValue N1, SDValue N2,
136 SmallVectorImpl<int> &Mask) const;
138 bool LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest,
139 SmallPtrSet<SDNode*, 32> &NodesLeadingTo);
141 void LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
144 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
145 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
146 SDNode *Node, bool isSigned);
147 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
148 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
149 RTLIB::Libcall Call_PPCF128);
150 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
151 RTLIB::Libcall Call_I8,
152 RTLIB::Libcall Call_I16,
153 RTLIB::Libcall Call_I32,
154 RTLIB::Libcall Call_I64,
155 RTLIB::Libcall Call_I128);
157 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, DebugLoc dl);
158 SDValue ExpandBUILD_VECTOR(SDNode *Node);
159 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
160 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
161 SmallVectorImpl<SDValue> &Results);
162 SDValue ExpandFCOPYSIGN(SDNode *Node);
163 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT,
165 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
167 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
170 SDValue ExpandBSWAP(SDValue Op, DebugLoc dl);
171 SDValue ExpandBitCount(unsigned Opc, SDValue Op, DebugLoc dl);
173 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
174 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
175 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
177 std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
179 void ExpandNode(SDNode *Node, SmallVectorImpl<SDValue> &Results);
180 void PromoteNode(SDNode *Node, SmallVectorImpl<SDValue> &Results);
184 /// ShuffleWithNarrowerEltType - Return a vector shuffle operation which
185 /// performs the same shuffe in terms of order or result bytes, but on a type
186 /// whose vector element type is narrower than the original shuffle type.
187 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
189 SelectionDAGLegalize::ShuffleWithNarrowerEltType(EVT NVT, EVT VT, DebugLoc dl,
190 SDValue N1, SDValue N2,
191 SmallVectorImpl<int> &Mask) const {
192 unsigned NumMaskElts = VT.getVectorNumElements();
193 unsigned NumDestElts = NVT.getVectorNumElements();
194 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
196 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
198 if (NumEltsGrowth == 1)
199 return DAG.getVectorShuffle(NVT, dl, N1, N2, &Mask[0]);
201 SmallVector<int, 8> NewMask;
202 for (unsigned i = 0; i != NumMaskElts; ++i) {
204 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
206 NewMask.push_back(-1);
208 NewMask.push_back(Idx * NumEltsGrowth + j);
211 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
212 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
213 return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]);
216 SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag,
217 CodeGenOpt::Level ol)
218 : TM(dag.getTarget()), TLI(dag.getTargetLoweringInfo()),
219 DAG(dag), OptLevel(ol),
220 ValueTypeActions(TLI.getValueTypeActions()) {
221 assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE &&
222 "Too many value types for ValueTypeActions to hold!");
225 void SelectionDAGLegalize::LegalizeDAG() {
226 LastCALLSEQ_END = DAG.getEntryNode();
228 // The legalize process is inherently a bottom-up recursive process (users
229 // legalize their uses before themselves). Given infinite stack space, we
230 // could just start legalizing on the root and traverse the whole graph. In
231 // practice however, this causes us to run out of stack space on large basic
232 // blocks. To avoid this problem, compute an ordering of the nodes where each
233 // node is only legalized after all of its operands are legalized.
234 DAG.AssignTopologicalOrder();
235 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
236 E = prior(DAG.allnodes_end()); I != llvm::next(E); ++I)
237 LegalizeOp(SDValue(I, 0));
239 // Finally, it's possible the root changed. Get the new root.
240 SDValue OldRoot = DAG.getRoot();
241 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
242 DAG.setRoot(LegalizedNodes[OldRoot]);
244 LegalizedNodes.clear();
246 // Remove dead nodes now.
247 DAG.RemoveDeadNodes();
251 /// FindCallEndFromCallStart - Given a chained node that is part of a call
252 /// sequence, find the CALLSEQ_END node that terminates the call sequence.
253 static SDNode *FindCallEndFromCallStart(SDNode *Node, int depth = 0) {
254 // Nested CALLSEQ_START/END constructs aren't yet legal,
255 // but we can DTRT and handle them correctly here.
256 if (Node->getOpcode() == ISD::CALLSEQ_START)
258 else if (Node->getOpcode() == ISD::CALLSEQ_END) {
263 if (Node->use_empty())
264 return 0; // No CallSeqEnd
266 // The chain is usually at the end.
267 SDValue TheChain(Node, Node->getNumValues()-1);
268 if (TheChain.getValueType() != MVT::Other) {
269 // Sometimes it's at the beginning.
270 TheChain = SDValue(Node, 0);
271 if (TheChain.getValueType() != MVT::Other) {
272 // Otherwise, hunt for it.
273 for (unsigned i = 1, e = Node->getNumValues(); i != e; ++i)
274 if (Node->getValueType(i) == MVT::Other) {
275 TheChain = SDValue(Node, i);
279 // Otherwise, we walked into a node without a chain.
280 if (TheChain.getValueType() != MVT::Other)
285 for (SDNode::use_iterator UI = Node->use_begin(),
286 E = Node->use_end(); UI != E; ++UI) {
288 // Make sure to only follow users of our token chain.
290 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i)
291 if (User->getOperand(i) == TheChain)
292 if (SDNode *Result = FindCallEndFromCallStart(User, depth))
298 /// FindCallStartFromCallEnd - Given a chained node that is part of a call
299 /// sequence, find the CALLSEQ_START node that initiates the call sequence.
300 static SDNode *FindCallStartFromCallEnd(SDNode *Node) {
302 assert(Node && "Didn't find callseq_start for a call??");
303 while (Node->getOpcode() != ISD::CALLSEQ_START || nested) {
304 Node = Node->getOperand(0).getNode();
305 assert(Node->getOperand(0).getValueType() == MVT::Other &&
306 "Node doesn't have a token chain argument!");
307 switch (Node->getOpcode()) {
310 case ISD::CALLSEQ_START:
315 case ISD::CALLSEQ_END:
323 /// LegalizeAllNodesNotLeadingTo - Recursively walk the uses of N, looking to
324 /// see if any uses can reach Dest. If no dest operands can get to dest,
325 /// legalize them, legalize ourself, and return false, otherwise, return true.
327 /// Keep track of the nodes we fine that actually do lead to Dest in
328 /// NodesLeadingTo. This avoids retraversing them exponential number of times.
330 bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest,
331 SmallPtrSet<SDNode*, 32> &NodesLeadingTo) {
332 if (N == Dest) return true; // N certainly leads to Dest :)
334 // If we've already processed this node and it does lead to Dest, there is no
335 // need to reprocess it.
336 if (NodesLeadingTo.count(N)) return true;
338 // If the first result of this node has been already legalized, then it cannot
340 if (LegalizedNodes.count(SDValue(N, 0))) return false;
342 // Okay, this node has not already been legalized. Check and legalize all
343 // operands. If none lead to Dest, then we can legalize this node.
344 bool OperandsLeadToDest = false;
345 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
346 OperandsLeadToDest |= // If an operand leads to Dest, so do we.
347 LegalizeAllNodesNotLeadingTo(N->getOperand(i).getNode(), Dest,
350 if (OperandsLeadToDest) {
351 NodesLeadingTo.insert(N);
355 // Okay, this node looks safe, legalize it and return false.
356 LegalizeOp(SDValue(N, 0));
360 /// ExpandConstantFP - Expands the ConstantFP node to an integer constant or
361 /// a load from the constant pool.
362 static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP,
363 SelectionDAG &DAG, const TargetLowering &TLI) {
365 DebugLoc dl = CFP->getDebugLoc();
367 // If a FP immediate is precise when represented as a float and if the
368 // target can do an extending load from float to double, we put it into
369 // the constant pool as a float, even if it's is statically typed as a
370 // double. This shrinks FP constants and canonicalizes them for targets where
371 // an FP extending load is the same cost as a normal load (such as on the x87
372 // fp stack or PPC FP unit).
373 EVT VT = CFP->getValueType(0);
374 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
376 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
377 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(),
378 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
383 while (SVT != MVT::f32) {
384 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
385 if (ConstantFPSDNode::isValueValidForType(SVT, CFP->getValueAPF()) &&
386 // Only do this if the target has a native EXTLOAD instruction from
388 TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) &&
389 TLI.ShouldShrinkFPConstant(OrigVT)) {
390 const Type *SType = SVT.getTypeForEVT(*DAG.getContext());
391 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
397 SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy());
398 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
400 return DAG.getExtLoad(ISD::EXTLOAD, dl, OrigVT,
402 CPIdx, MachinePointerInfo::getConstantPool(),
403 VT, false, false, Alignment);
404 return DAG.getLoad(OrigVT, dl, DAG.getEntryNode(), CPIdx,
405 MachinePointerInfo::getConstantPool(), false, false,
409 /// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores.
411 SDValue ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG,
412 const TargetLowering &TLI) {
413 SDValue Chain = ST->getChain();
414 SDValue Ptr = ST->getBasePtr();
415 SDValue Val = ST->getValue();
416 EVT VT = Val.getValueType();
417 int Alignment = ST->getAlignment();
418 DebugLoc dl = ST->getDebugLoc();
419 if (ST->getMemoryVT().isFloatingPoint() ||
420 ST->getMemoryVT().isVector()) {
421 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
422 if (TLI.isTypeLegal(intVT)) {
423 // Expand to a bitconvert of the value to the integer type of the
424 // same size, then a (misaligned) int store.
425 // FIXME: Does not handle truncating floating point stores!
426 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
427 return DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
428 ST->isVolatile(), ST->isNonTemporal(), Alignment);
430 // Do a (aligned) store to a stack slot, then copy from the stack slot
431 // to the final destination using (unaligned) integer loads and stores.
432 EVT StoredVT = ST->getMemoryVT();
434 TLI.getRegisterType(*DAG.getContext(),
435 EVT::getIntegerVT(*DAG.getContext(),
436 StoredVT.getSizeInBits()));
437 unsigned StoredBytes = StoredVT.getSizeInBits() / 8;
438 unsigned RegBytes = RegVT.getSizeInBits() / 8;
439 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
441 // Make sure the stack slot is also aligned for the register type.
442 SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT);
444 // Perform the original store, only redirected to the stack slot.
445 SDValue Store = DAG.getTruncStore(Chain, dl,
446 Val, StackPtr, MachinePointerInfo(),
447 StoredVT, false, false, 0);
448 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy());
449 SmallVector<SDValue, 8> Stores;
452 // Do all but one copies using the full register width.
453 for (unsigned i = 1; i < NumRegs; i++) {
454 // Load one integer register's worth from the stack slot.
455 SDValue Load = DAG.getLoad(RegVT, dl, Store, StackPtr,
456 MachinePointerInfo(),
458 // Store it to the final location. Remember the store.
459 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,
460 ST->getPointerInfo().getWithOffset(Offset),
461 ST->isVolatile(), ST->isNonTemporal(),
462 MinAlign(ST->getAlignment(), Offset)));
463 // Increment the pointers.
465 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
467 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
470 // The last store may be partial. Do a truncating store. On big-endian
471 // machines this requires an extending load from the stack slot to ensure
472 // that the bits are in the right place.
473 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
474 8 * (StoredBytes - Offset));
476 // Load from the stack slot.
477 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
478 MachinePointerInfo(),
479 MemVT, false, false, 0);
481 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,
483 .getWithOffset(Offset),
484 MemVT, ST->isVolatile(),
486 MinAlign(ST->getAlignment(), Offset)));
487 // The order of the stores doesn't matter - say it with a TokenFactor.
488 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0],
492 assert(ST->getMemoryVT().isInteger() &&
493 !ST->getMemoryVT().isVector() &&
494 "Unaligned store of unknown type.");
495 // Get the half-size VT
496 EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext());
497 int NumBits = NewStoredVT.getSizeInBits();
498 int IncrementSize = NumBits / 8;
500 // Divide the stored value in two parts.
501 SDValue ShiftAmount = DAG.getConstant(NumBits,
502 TLI.getShiftAmountTy(Val.getValueType()));
504 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
506 // Store the two parts
507 SDValue Store1, Store2;
508 Store1 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Lo:Hi, Ptr,
509 ST->getPointerInfo(), NewStoredVT,
510 ST->isVolatile(), ST->isNonTemporal(), Alignment);
511 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
512 DAG.getConstant(IncrementSize, TLI.getPointerTy()));
513 Alignment = MinAlign(Alignment, IncrementSize);
514 Store2 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Hi:Lo, Ptr,
515 ST->getPointerInfo().getWithOffset(IncrementSize),
516 NewStoredVT, ST->isVolatile(), ST->isNonTemporal(),
519 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
522 /// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads.
524 SDValue ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG,
525 const TargetLowering &TLI) {
526 SDValue Chain = LD->getChain();
527 SDValue Ptr = LD->getBasePtr();
528 EVT VT = LD->getValueType(0);
529 EVT LoadedVT = LD->getMemoryVT();
530 DebugLoc dl = LD->getDebugLoc();
531 if (VT.isFloatingPoint() || VT.isVector()) {
532 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
533 if (TLI.isTypeLegal(intVT)) {
534 // Expand to a (misaligned) integer load of the same size,
535 // then bitconvert to floating point or vector.
536 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, LD->getPointerInfo(),
538 LD->isNonTemporal(), LD->getAlignment());
539 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
540 if (VT.isFloatingPoint() && LoadedVT != VT)
541 Result = DAG.getNode(ISD::FP_EXTEND, dl, VT, Result);
543 SDValue Ops[] = { Result, Chain };
544 return DAG.getMergeValues(Ops, 2, dl);
547 // Copy the value to a (aligned) stack slot using (unaligned) integer
548 // loads and stores, then do a (aligned) load from the stack slot.
549 EVT RegVT = TLI.getRegisterType(*DAG.getContext(), intVT);
550 unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8;
551 unsigned RegBytes = RegVT.getSizeInBits() / 8;
552 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
554 // Make sure the stack slot is also aligned for the register type.
555 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
557 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy());
558 SmallVector<SDValue, 8> Stores;
559 SDValue StackPtr = StackBase;
562 // Do all but one copies using the full register width.
563 for (unsigned i = 1; i < NumRegs; i++) {
564 // Load one integer register's worth from the original location.
565 SDValue Load = DAG.getLoad(RegVT, dl, Chain, Ptr,
566 LD->getPointerInfo().getWithOffset(Offset),
567 LD->isVolatile(), LD->isNonTemporal(),
568 MinAlign(LD->getAlignment(), Offset));
569 // Follow the load with a store to the stack slot. Remember the store.
570 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, StackPtr,
571 MachinePointerInfo(), false, false, 0));
572 // Increment the pointers.
574 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
575 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
579 // The last copy may be partial. Do an extending load.
580 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
581 8 * (LoadedBytes - Offset));
582 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
583 LD->getPointerInfo().getWithOffset(Offset),
584 MemVT, LD->isVolatile(),
586 MinAlign(LD->getAlignment(), Offset));
587 // Follow the load with a store to the stack slot. Remember the store.
588 // On big-endian machines this requires a truncating store to ensure
589 // that the bits end up in the right place.
590 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, StackPtr,
591 MachinePointerInfo(), MemVT,
594 // The order of the stores doesn't matter - say it with a TokenFactor.
595 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0],
598 // Finally, perform the original load only redirected to the stack slot.
599 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
600 MachinePointerInfo(), LoadedVT, false, false, 0);
602 // Callers expect a MERGE_VALUES node.
603 SDValue Ops[] = { Load, TF };
604 return DAG.getMergeValues(Ops, 2, dl);
606 assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
607 "Unaligned load of unsupported type.");
609 // Compute the new VT that is half the size of the old one. This is an
611 unsigned NumBits = LoadedVT.getSizeInBits();
613 NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
616 unsigned Alignment = LD->getAlignment();
617 unsigned IncrementSize = NumBits / 8;
618 ISD::LoadExtType HiExtType = LD->getExtensionType();
620 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
621 if (HiExtType == ISD::NON_EXTLOAD)
622 HiExtType = ISD::ZEXTLOAD;
624 // Load the value in two parts
626 if (TLI.isLittleEndian()) {
627 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
628 NewLoadedVT, LD->isVolatile(),
629 LD->isNonTemporal(), Alignment);
630 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
631 DAG.getConstant(IncrementSize, TLI.getPointerTy()));
632 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
633 LD->getPointerInfo().getWithOffset(IncrementSize),
634 NewLoadedVT, LD->isVolatile(),
635 LD->isNonTemporal(), MinAlign(Alignment,IncrementSize));
637 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
638 NewLoadedVT, LD->isVolatile(),
639 LD->isNonTemporal(), Alignment);
640 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
641 DAG.getConstant(IncrementSize, TLI.getPointerTy()));
642 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
643 LD->getPointerInfo().getWithOffset(IncrementSize),
644 NewLoadedVT, LD->isVolatile(),
645 LD->isNonTemporal(), MinAlign(Alignment,IncrementSize));
648 // aggregate the two parts
649 SDValue ShiftAmount = DAG.getConstant(NumBits,
650 TLI.getShiftAmountTy(Hi.getValueType()));
651 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
652 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
654 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
657 SDValue Ops[] = { Result, TF };
658 return DAG.getMergeValues(Ops, 2, dl);
661 /// PerformInsertVectorEltInMemory - Some target cannot handle a variable
662 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
663 /// is necessary to spill the vector being inserted into to memory, perform
664 /// the insert there, and then read the result back.
665 SDValue SelectionDAGLegalize::
666 PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
672 // If the target doesn't support this, we have to spill the input vector
673 // to a temporary stack slot, update the element, then reload it. This is
674 // badness. We could also load the value into a vector register (either
675 // with a "move to register" or "extload into register" instruction, then
676 // permute it into place, if the idx is a constant and if the idx is
677 // supported by the target.
678 EVT VT = Tmp1.getValueType();
679 EVT EltVT = VT.getVectorElementType();
680 EVT IdxVT = Tmp3.getValueType();
681 EVT PtrVT = TLI.getPointerTy();
682 SDValue StackPtr = DAG.CreateStackTemporary(VT);
684 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
687 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Tmp1, StackPtr,
688 MachinePointerInfo::getFixedStack(SPFI),
691 // Truncate or zero extend offset to target pointer type.
692 unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
693 Tmp3 = DAG.getNode(CastOpc, dl, PtrVT, Tmp3);
694 // Add the offset to the index.
695 unsigned EltSize = EltVT.getSizeInBits()/8;
696 Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT));
697 SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr);
698 // Store the scalar value.
699 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT,
701 // Load the updated vector.
702 return DAG.getLoad(VT, dl, Ch, StackPtr,
703 MachinePointerInfo::getFixedStack(SPFI), false, false, 0);
707 SDValue SelectionDAGLegalize::
708 ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, DebugLoc dl) {
709 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
710 // SCALAR_TO_VECTOR requires that the type of the value being inserted
711 // match the element type of the vector being created, except for
712 // integers in which case the inserted value can be over width.
713 EVT EltVT = Vec.getValueType().getVectorElementType();
714 if (Val.getValueType() == EltVT ||
715 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
716 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
717 Vec.getValueType(), Val);
719 unsigned NumElts = Vec.getValueType().getVectorNumElements();
720 // We generate a shuffle of InVec and ScVec, so the shuffle mask
721 // should be 0,1,2,3,4,5... with the appropriate element replaced with
723 SmallVector<int, 8> ShufOps;
724 for (unsigned i = 0; i != NumElts; ++i)
725 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
727 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec,
731 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
734 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
735 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
736 // FIXME: We shouldn't do this for TargetConstantFP's.
737 // FIXME: move this to the DAG Combiner! Note that we can't regress due
738 // to phase ordering between legalized code and the dag combiner. This
739 // probably means that we need to integrate dag combiner and legalizer
741 // We generally can't do this one for long doubles.
742 SDValue Tmp1 = ST->getChain();
743 SDValue Tmp2 = ST->getBasePtr();
745 unsigned Alignment = ST->getAlignment();
746 bool isVolatile = ST->isVolatile();
747 bool isNonTemporal = ST->isNonTemporal();
748 DebugLoc dl = ST->getDebugLoc();
749 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
750 if (CFP->getValueType(0) == MVT::f32 &&
751 getTypeAction(MVT::i32) == Legal) {
752 Tmp3 = DAG.getConstant(CFP->getValueAPF().
753 bitcastToAPInt().zextOrTrunc(32),
755 return DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(),
756 isVolatile, isNonTemporal, Alignment);
759 if (CFP->getValueType(0) == MVT::f64) {
760 // If this target supports 64-bit registers, do a single 64-bit store.
761 if (getTypeAction(MVT::i64) == Legal) {
762 Tmp3 = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
763 zextOrTrunc(64), MVT::i64);
764 return DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(),
765 isVolatile, isNonTemporal, Alignment);
768 if (getTypeAction(MVT::i32) == Legal && !ST->isVolatile()) {
769 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
770 // stores. If the target supports neither 32- nor 64-bits, this
771 // xform is certainly not worth it.
772 const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt();
773 SDValue Lo = DAG.getConstant(IntVal.trunc(32), MVT::i32);
774 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32);
775 if (TLI.isBigEndian()) std::swap(Lo, Hi);
777 Lo = DAG.getStore(Tmp1, dl, Lo, Tmp2, ST->getPointerInfo(), isVolatile,
778 isNonTemporal, Alignment);
779 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
780 DAG.getIntPtrConstant(4));
781 Hi = DAG.getStore(Tmp1, dl, Hi, Tmp2,
782 ST->getPointerInfo().getWithOffset(4),
783 isVolatile, isNonTemporal, MinAlign(Alignment, 4U));
785 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
792 /// LegalizeOp - We know that the specified value has a legal type, and
793 /// that its operands are legal. Now ensure that the operation itself
794 /// is legal, recursively ensuring that the operands' operations remain
796 SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
797 if (Op.getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
800 SDNode *Node = Op.getNode();
801 DebugLoc dl = Node->getDebugLoc();
803 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
804 assert(getTypeAction(Node->getValueType(i)) == Legal &&
805 "Unexpected illegal type!");
807 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
808 assert((isTypeLegal(Node->getOperand(i).getValueType()) ||
809 Node->getOperand(i).getOpcode() == ISD::TargetConstant) &&
810 "Unexpected illegal type!");
812 // Note that LegalizeOp may be reentered even from single-use nodes, which
813 // means that we always must cache transformed nodes.
814 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
815 if (I != LegalizedNodes.end()) return I->second;
817 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
819 bool isCustom = false;
821 // Figure out the correct action; the way to query this varies by opcode
822 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
823 bool SimpleFinishLegalizing = true;
824 switch (Node->getOpcode()) {
825 case ISD::INTRINSIC_W_CHAIN:
826 case ISD::INTRINSIC_WO_CHAIN:
827 case ISD::INTRINSIC_VOID:
830 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
832 case ISD::SINT_TO_FP:
833 case ISD::UINT_TO_FP:
834 case ISD::EXTRACT_VECTOR_ELT:
835 Action = TLI.getOperationAction(Node->getOpcode(),
836 Node->getOperand(0).getValueType());
838 case ISD::FP_ROUND_INREG:
839 case ISD::SIGN_EXTEND_INREG: {
840 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
841 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
847 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
848 Node->getOpcode() == ISD::SETCC ? 2 : 1;
849 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
850 EVT OpVT = Node->getOperand(CompareOperand).getValueType();
851 ISD::CondCode CCCode =
852 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
853 Action = TLI.getCondCodeAction(CCCode, OpVT);
854 if (Action == TargetLowering::Legal) {
855 if (Node->getOpcode() == ISD::SELECT_CC)
856 Action = TLI.getOperationAction(Node->getOpcode(),
857 Node->getValueType(0));
859 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
865 // FIXME: Model these properly. LOAD and STORE are complicated, and
866 // STORE expects the unlegalized operand in some cases.
867 SimpleFinishLegalizing = false;
869 case ISD::CALLSEQ_START:
870 case ISD::CALLSEQ_END:
871 // FIXME: This shouldn't be necessary. These nodes have special properties
872 // dealing with the recursive nature of legalization. Removing this
873 // special case should be done as part of making LegalizeDAG non-recursive.
874 SimpleFinishLegalizing = false;
876 case ISD::EXTRACT_ELEMENT:
877 case ISD::FLT_ROUNDS_:
885 case ISD::MERGE_VALUES:
887 case ISD::FRAME_TO_ARGS_OFFSET:
888 case ISD::EH_SJLJ_SETJMP:
889 case ISD::EH_SJLJ_LONGJMP:
890 case ISD::EH_SJLJ_DISPATCHSETUP:
891 // These operations lie about being legal: when they claim to be legal,
892 // they should actually be expanded.
893 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
894 if (Action == TargetLowering::Legal)
895 Action = TargetLowering::Expand;
897 case ISD::TRAMPOLINE:
899 case ISD::RETURNADDR:
900 // These operations lie about being legal: when they claim to be legal,
901 // they should actually be custom-lowered.
902 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
903 if (Action == TargetLowering::Legal)
904 Action = TargetLowering::Custom;
906 case ISD::BUILD_VECTOR:
907 // A weird case: legalization for BUILD_VECTOR never legalizes the
909 // FIXME: This really sucks... changing it isn't semantically incorrect,
910 // but it massively pessimizes the code for floating-point BUILD_VECTORs
911 // because ConstantFP operands get legalized into constant pool loads
912 // before the BUILD_VECTOR code can see them. It doesn't usually bite,
913 // though, because BUILD_VECTORS usually get lowered into other nodes
914 // which get legalized properly.
915 SimpleFinishLegalizing = false;
918 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
919 Action = TargetLowering::Legal;
921 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
926 if (SimpleFinishLegalizing) {
927 SmallVector<SDValue, 8> Ops, ResultVals;
928 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
929 Ops.push_back(LegalizeOp(Node->getOperand(i)));
930 switch (Node->getOpcode()) {
937 // Branches tweak the chain to include LastCALLSEQ_END
938 Ops[0] = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ops[0],
940 Ops[0] = LegalizeOp(Ops[0]);
941 LastCALLSEQ_END = DAG.getEntryNode();
948 // Legalizing shifts/rotates requires adjusting the shift amount
949 // to the appropriate width.
950 if (!Ops[1].getValueType().isVector())
951 Ops[1] = LegalizeOp(DAG.getShiftAmountOperand(Ops[0].getValueType(),
957 // Legalizing shifts/rotates requires adjusting the shift amount
958 // to the appropriate width.
959 if (!Ops[2].getValueType().isVector())
960 Ops[2] = LegalizeOp(DAG.getShiftAmountOperand(Ops[0].getValueType(),
965 Result = SDValue(DAG.UpdateNodeOperands(Result.getNode(), Ops.data(),
968 case TargetLowering::Legal:
969 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
970 ResultVals.push_back(Result.getValue(i));
972 case TargetLowering::Custom:
973 // FIXME: The handling for custom lowering with multiple results is
975 Tmp1 = TLI.LowerOperation(Result, DAG);
976 if (Tmp1.getNode()) {
977 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
979 ResultVals.push_back(Tmp1);
981 ResultVals.push_back(Tmp1.getValue(i));
987 case TargetLowering::Expand:
988 ExpandNode(Result.getNode(), ResultVals);
990 case TargetLowering::Promote:
991 PromoteNode(Result.getNode(), ResultVals);
994 if (!ResultVals.empty()) {
995 for (unsigned i = 0, e = ResultVals.size(); i != e; ++i) {
996 if (ResultVals[i] != SDValue(Node, i))
997 ResultVals[i] = LegalizeOp(ResultVals[i]);
998 AddLegalizedOperand(SDValue(Node, i), ResultVals[i]);
1000 return ResultVals[Op.getResNo()];
1004 switch (Node->getOpcode()) {
1011 assert(0 && "Do not know how to legalize this operator!");
1013 case ISD::BUILD_VECTOR:
1014 switch (TLI.getOperationAction(ISD::BUILD_VECTOR, Node->getValueType(0))) {
1015 default: assert(0 && "This action is not supported yet!");
1016 case TargetLowering::Custom:
1017 Tmp3 = TLI.LowerOperation(Result, DAG);
1018 if (Tmp3.getNode()) {
1023 case TargetLowering::Expand:
1024 Result = ExpandBUILD_VECTOR(Result.getNode());
1028 case ISD::CALLSEQ_START: {
1029 static int depth = 0;
1030 SDNode *CallEnd = FindCallEndFromCallStart(Node);
1032 // Recursively Legalize all of the inputs of the call end that do not lead
1033 // to this call start. This ensures that any libcalls that need be inserted
1034 // are inserted *before* the CALLSEQ_START.
1035 {SmallPtrSet<SDNode*, 32> NodesLeadingTo;
1036 for (unsigned i = 0, e = CallEnd->getNumOperands(); i != e; ++i)
1037 LegalizeAllNodesNotLeadingTo(CallEnd->getOperand(i).getNode(), Node,
1041 // Now that we have legalized all of the inputs (which may have inserted
1042 // libcalls), create the new CALLSEQ_START node.
1043 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
1045 // Merge in the last call to ensure that this call starts after the last
1047 if (LastCALLSEQ_END.getOpcode() != ISD::EntryToken && depth == 0) {
1048 Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1049 Tmp1, LastCALLSEQ_END);
1050 Tmp1 = LegalizeOp(Tmp1);
1053 // Do not try to legalize the target-specific arguments (#1+).
1054 if (Tmp1 != Node->getOperand(0)) {
1055 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end());
1057 Result = SDValue(DAG.UpdateNodeOperands(Result.getNode(), &Ops[0],
1058 Ops.size()), Result.getResNo());
1061 // Remember that the CALLSEQ_START is legalized.
1062 AddLegalizedOperand(Op.getValue(0), Result);
1063 if (Node->getNumValues() == 2) // If this has a flag result, remember it.
1064 AddLegalizedOperand(Op.getValue(1), Result.getValue(1));
1066 // Now that the callseq_start and all of the non-call nodes above this call
1067 // sequence have been legalized, legalize the call itself. During this
1068 // process, no libcalls can/will be inserted, guaranteeing that no calls
1071 SDValue Saved_LastCALLSEQ_END = LastCALLSEQ_END ;
1072 // Note that we are selecting this call!
1073 LastCALLSEQ_END = SDValue(CallEnd, 0);
1076 // Legalize the call, starting from the CALLSEQ_END.
1077 LegalizeOp(LastCALLSEQ_END);
1079 assert(depth >= 0 && "Un-matched CALLSEQ_START?");
1081 LastCALLSEQ_END = Saved_LastCALLSEQ_END;
1084 case ISD::CALLSEQ_END:
1085 // If the CALLSEQ_START node hasn't been legalized first, legalize it. This
1086 // will cause this node to be legalized as well as handling libcalls right.
1087 if (LastCALLSEQ_END.getNode() != Node) {
1088 LegalizeOp(SDValue(FindCallStartFromCallEnd(Node), 0));
1089 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
1090 assert(I != LegalizedNodes.end() &&
1091 "Legalizing the call start should have legalized this node!");
1095 // Otherwise, the call start has been legalized and everything is going
1096 // according to plan. Just legalize ourselves normally here.
1097 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
1098 // Do not try to legalize the target-specific arguments (#1+), except for
1099 // an optional flag input.
1100 if (Node->getOperand(Node->getNumOperands()-1).getValueType() != MVT::Glue){
1101 if (Tmp1 != Node->getOperand(0)) {
1102 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end());
1104 Result = SDValue(DAG.UpdateNodeOperands(Result.getNode(),
1105 &Ops[0], Ops.size()),
1109 Tmp2 = LegalizeOp(Node->getOperand(Node->getNumOperands()-1));
1110 if (Tmp1 != Node->getOperand(0) ||
1111 Tmp2 != Node->getOperand(Node->getNumOperands()-1)) {
1112 SmallVector<SDValue, 8> Ops(Node->op_begin(), Node->op_end());
1115 Result = SDValue(DAG.UpdateNodeOperands(Result.getNode(),
1116 &Ops[0], Ops.size()),
1120 // This finishes up call legalization.
1121 // If the CALLSEQ_END node has a flag, remember that we legalized it.
1122 AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0));
1123 if (Node->getNumValues() == 2)
1124 AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1));
1125 return Result.getValue(Op.getResNo());
1127 LoadSDNode *LD = cast<LoadSDNode>(Node);
1128 Tmp1 = LegalizeOp(LD->getChain()); // Legalize the chain.
1129 Tmp2 = LegalizeOp(LD->getBasePtr()); // Legalize the base pointer.
1131 ISD::LoadExtType ExtType = LD->getExtensionType();
1132 if (ExtType == ISD::NON_EXTLOAD) {
1133 EVT VT = Node->getValueType(0);
1134 Result = SDValue(DAG.UpdateNodeOperands(Result.getNode(),
1135 Tmp1, Tmp2, LD->getOffset()),
1137 Tmp3 = Result.getValue(0);
1138 Tmp4 = Result.getValue(1);
1140 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
1141 default: assert(0 && "This action is not supported yet!");
1142 case TargetLowering::Legal:
1143 // If this is an unaligned load and the target doesn't support it,
1145 if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) {
1146 const Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
1147 unsigned ABIAlignment = TLI.getTargetData()->getABITypeAlignment(Ty);
1148 if (LD->getAlignment() < ABIAlignment){
1149 Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.getNode()),
1151 Tmp3 = Result.getOperand(0);
1152 Tmp4 = Result.getOperand(1);
1153 Tmp3 = LegalizeOp(Tmp3);
1154 Tmp4 = LegalizeOp(Tmp4);
1158 case TargetLowering::Custom:
1159 Tmp1 = TLI.LowerOperation(Tmp3, DAG);
1160 if (Tmp1.getNode()) {
1161 Tmp3 = LegalizeOp(Tmp1);
1162 Tmp4 = LegalizeOp(Tmp1.getValue(1));
1165 case TargetLowering::Promote: {
1166 // Only promote a load of vector type to another.
1167 assert(VT.isVector() && "Cannot promote this load!");
1168 // Change base type to a different vector type.
1169 EVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
1171 Tmp1 = DAG.getLoad(NVT, dl, Tmp1, Tmp2, LD->getPointerInfo(),
1172 LD->isVolatile(), LD->isNonTemporal(),
1173 LD->getAlignment());
1174 Tmp3 = LegalizeOp(DAG.getNode(ISD::BITCAST, dl, VT, Tmp1));
1175 Tmp4 = LegalizeOp(Tmp1.getValue(1));
1179 // Since loads produce two values, make sure to remember that we
1180 // legalized both of them.
1181 AddLegalizedOperand(SDValue(Node, 0), Tmp3);
1182 AddLegalizedOperand(SDValue(Node, 1), Tmp4);
1183 return Op.getResNo() ? Tmp4 : Tmp3;
1186 EVT SrcVT = LD->getMemoryVT();
1187 unsigned SrcWidth = SrcVT.getSizeInBits();
1188 unsigned Alignment = LD->getAlignment();
1189 bool isVolatile = LD->isVolatile();
1190 bool isNonTemporal = LD->isNonTemporal();
1192 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
1193 // Some targets pretend to have an i1 loading operation, and actually
1194 // load an i8. This trick is correct for ZEXTLOAD because the top 7
1195 // bits are guaranteed to be zero; it helps the optimizers understand
1196 // that these bits are zero. It is also useful for EXTLOAD, since it
1197 // tells the optimizers that those bits are undefined. It would be
1198 // nice to have an effective generic way of getting these benefits...
1199 // Until such a way is found, don't insist on promoting i1 here.
1200 (SrcVT != MVT::i1 ||
1201 TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) {
1202 // Promote to a byte-sized load if not loading an integral number of
1203 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
1204 unsigned NewWidth = SrcVT.getStoreSizeInBits();
1205 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
1208 // The extra bits are guaranteed to be zero, since we stored them that
1209 // way. A zext load from NVT thus automatically gives zext from SrcVT.
1211 ISD::LoadExtType NewExtType =
1212 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
1214 Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
1215 Tmp1, Tmp2, LD->getPointerInfo(),
1216 NVT, isVolatile, isNonTemporal, Alignment);
1218 Ch = Result.getValue(1); // The chain.
1220 if (ExtType == ISD::SEXTLOAD)
1221 // Having the top bits zero doesn't help when sign extending.
1222 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
1223 Result.getValueType(),
1224 Result, DAG.getValueType(SrcVT));
1225 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
1226 // All the top bits are guaranteed to be zero - inform the optimizers.
1227 Result = DAG.getNode(ISD::AssertZext, dl,
1228 Result.getValueType(), Result,
1229 DAG.getValueType(SrcVT));
1231 Tmp1 = LegalizeOp(Result);
1232 Tmp2 = LegalizeOp(Ch);
1233 } else if (SrcWidth & (SrcWidth - 1)) {
1234 // If not loading a power-of-2 number of bits, expand as two loads.
1235 assert(!SrcVT.isVector() && "Unsupported extload!");
1236 unsigned RoundWidth = 1 << Log2_32(SrcWidth);
1237 assert(RoundWidth < SrcWidth);
1238 unsigned ExtraWidth = SrcWidth - RoundWidth;
1239 assert(ExtraWidth < RoundWidth);
1240 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
1241 "Load size not an integral number of bytes!");
1242 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
1243 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
1245 unsigned IncrementSize;
1247 if (TLI.isLittleEndian()) {
1248 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
1249 // Load the bottom RoundWidth bits.
1250 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0),
1252 LD->getPointerInfo(), RoundVT, isVolatile,
1253 isNonTemporal, Alignment);
1255 // Load the remaining ExtraWidth bits.
1256 IncrementSize = RoundWidth / 8;
1257 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
1258 DAG.getIntPtrConstant(IncrementSize));
1259 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Tmp1, Tmp2,
1260 LD->getPointerInfo().getWithOffset(IncrementSize),
1261 ExtraVT, isVolatile, isNonTemporal,
1262 MinAlign(Alignment, IncrementSize));
1264 // Build a factor node to remember that this load is independent of
1266 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1269 // Move the top bits to the right place.
1270 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi,
1271 DAG.getConstant(RoundWidth,
1272 TLI.getShiftAmountTy(Hi.getValueType())));
1274 // Join the hi and lo parts.
1275 Result = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
1277 // Big endian - avoid unaligned loads.
1278 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
1279 // Load the top RoundWidth bits.
1280 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Tmp1, Tmp2,
1281 LD->getPointerInfo(), RoundVT, isVolatile,
1282 isNonTemporal, Alignment);
1284 // Load the remaining ExtraWidth bits.
1285 IncrementSize = RoundWidth / 8;
1286 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
1287 DAG.getIntPtrConstant(IncrementSize));
1288 Lo = DAG.getExtLoad(ISD::ZEXTLOAD,
1289 dl, Node->getValueType(0), Tmp1, Tmp2,
1290 LD->getPointerInfo().getWithOffset(IncrementSize),
1291 ExtraVT, isVolatile, isNonTemporal,
1292 MinAlign(Alignment, IncrementSize));
1294 // Build a factor node to remember that this load is independent of
1296 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1299 // Move the top bits to the right place.
1300 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi,
1301 DAG.getConstant(ExtraWidth,
1302 TLI.getShiftAmountTy(Hi.getValueType())));
1304 // Join the hi and lo parts.
1305 Result = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
1308 Tmp1 = LegalizeOp(Result);
1309 Tmp2 = LegalizeOp(Ch);
1311 switch (TLI.getLoadExtAction(ExtType, SrcVT)) {
1312 default: assert(0 && "This action is not supported yet!");
1313 case TargetLowering::Custom:
1316 case TargetLowering::Legal:
1317 Result = SDValue(DAG.UpdateNodeOperands(Result.getNode(),
1318 Tmp1, Tmp2, LD->getOffset()),
1320 Tmp1 = Result.getValue(0);
1321 Tmp2 = Result.getValue(1);
1324 Tmp3 = TLI.LowerOperation(Result, DAG);
1325 if (Tmp3.getNode()) {
1326 Tmp1 = LegalizeOp(Tmp3);
1327 Tmp2 = LegalizeOp(Tmp3.getValue(1));
1330 // If this is an unaligned load and the target doesn't support it,
1332 if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) {
1334 LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
1335 unsigned ABIAlignment =
1336 TLI.getTargetData()->getABITypeAlignment(Ty);
1337 if (LD->getAlignment() < ABIAlignment){
1338 Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.getNode()),
1340 Tmp1 = Result.getOperand(0);
1341 Tmp2 = Result.getOperand(1);
1342 Tmp1 = LegalizeOp(Tmp1);
1343 Tmp2 = LegalizeOp(Tmp2);
1348 case TargetLowering::Expand:
1349 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, SrcVT) && isTypeLegal(SrcVT)) {
1350 SDValue Load = DAG.getLoad(SrcVT, dl, Tmp1, Tmp2,
1351 LD->getPointerInfo(),
1352 LD->isVolatile(), LD->isNonTemporal(),
1353 LD->getAlignment());
1357 ExtendOp = (SrcVT.isFloatingPoint() ?
1358 ISD::FP_EXTEND : ISD::ANY_EXTEND);
1360 case ISD::SEXTLOAD: ExtendOp = ISD::SIGN_EXTEND; break;
1361 case ISD::ZEXTLOAD: ExtendOp = ISD::ZERO_EXTEND; break;
1362 default: llvm_unreachable("Unexpected extend load type!");
1364 Result = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
1365 Tmp1 = LegalizeOp(Result); // Relegalize new nodes.
1366 Tmp2 = LegalizeOp(Load.getValue(1));
1369 // FIXME: This does not work for vectors on most targets. Sign- and
1370 // zero-extend operations are currently folded into extending loads,
1371 // whether they are legal or not, and then we end up here without any
1372 // support for legalizing them.
1373 assert(ExtType != ISD::EXTLOAD &&
1374 "EXTLOAD should always be supported!");
1375 // Turn the unsupported load into an EXTLOAD followed by an explicit
1376 // zero/sign extend inreg.
1377 Result = DAG.getExtLoad(ISD::EXTLOAD, dl, Node->getValueType(0),
1378 Tmp1, Tmp2, LD->getPointerInfo(), SrcVT,
1379 LD->isVolatile(), LD->isNonTemporal(),
1380 LD->getAlignment());
1382 if (ExtType == ISD::SEXTLOAD)
1383 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
1384 Result.getValueType(),
1385 Result, DAG.getValueType(SrcVT));
1387 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
1388 Tmp1 = LegalizeOp(ValRes); // Relegalize new nodes.
1389 Tmp2 = LegalizeOp(Result.getValue(1)); // Relegalize new nodes.
1394 // Since loads produce two values, make sure to remember that we legalized
1396 AddLegalizedOperand(SDValue(Node, 0), Tmp1);
1397 AddLegalizedOperand(SDValue(Node, 1), Tmp2);
1398 return Op.getResNo() ? Tmp2 : Tmp1;
1401 StoreSDNode *ST = cast<StoreSDNode>(Node);
1402 Tmp1 = LegalizeOp(ST->getChain()); // Legalize the chain.
1403 Tmp2 = LegalizeOp(ST->getBasePtr()); // Legalize the pointer.
1404 unsigned Alignment = ST->getAlignment();
1405 bool isVolatile = ST->isVolatile();
1406 bool isNonTemporal = ST->isNonTemporal();
1408 if (!ST->isTruncatingStore()) {
1409 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
1410 Result = SDValue(OptStore, 0);
1415 Tmp3 = LegalizeOp(ST->getValue());
1416 Result = SDValue(DAG.UpdateNodeOperands(Result.getNode(),
1421 EVT VT = Tmp3.getValueType();
1422 switch (TLI.getOperationAction(ISD::STORE, VT)) {
1423 default: assert(0 && "This action is not supported yet!");
1424 case TargetLowering::Legal:
1425 // If this is an unaligned store and the target doesn't support it,
1427 if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) {
1428 const Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
1429 unsigned ABIAlignment= TLI.getTargetData()->getABITypeAlignment(Ty);
1430 if (ST->getAlignment() < ABIAlignment)
1431 Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()),
1435 case TargetLowering::Custom:
1436 Tmp1 = TLI.LowerOperation(Result, DAG);
1437 if (Tmp1.getNode()) Result = Tmp1;
1439 case TargetLowering::Promote:
1440 assert(VT.isVector() && "Unknown legal promote case!");
1441 Tmp3 = DAG.getNode(ISD::BITCAST, dl,
1442 TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3);
1443 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2,
1444 ST->getPointerInfo(), isVolatile,
1445 isNonTemporal, Alignment);
1451 Tmp3 = LegalizeOp(ST->getValue());
1453 EVT StVT = ST->getMemoryVT();
1454 unsigned StWidth = StVT.getSizeInBits();
1456 if (StWidth != StVT.getStoreSizeInBits()) {
1457 // Promote to a byte-sized store with upper bits zero if not
1458 // storing an integral number of bytes. For example, promote
1459 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
1460 EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
1461 StVT.getStoreSizeInBits());
1462 Tmp3 = DAG.getZeroExtendInReg(Tmp3, dl, StVT);
1463 Result = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(),
1464 NVT, isVolatile, isNonTemporal, Alignment);
1465 } else if (StWidth & (StWidth - 1)) {
1466 // If not storing a power-of-2 number of bits, expand as two stores.
1467 assert(!StVT.isVector() && "Unsupported truncstore!");
1468 unsigned RoundWidth = 1 << Log2_32(StWidth);
1469 assert(RoundWidth < StWidth);
1470 unsigned ExtraWidth = StWidth - RoundWidth;
1471 assert(ExtraWidth < RoundWidth);
1472 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
1473 "Store size not an integral number of bytes!");
1474 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
1475 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
1477 unsigned IncrementSize;
1479 if (TLI.isLittleEndian()) {
1480 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
1481 // Store the bottom RoundWidth bits.
1482 Lo = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(),
1484 isVolatile, isNonTemporal, Alignment);
1486 // Store the remaining ExtraWidth bits.
1487 IncrementSize = RoundWidth / 8;
1488 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
1489 DAG.getIntPtrConstant(IncrementSize));
1490 Hi = DAG.getNode(ISD::SRL, dl, Tmp3.getValueType(), Tmp3,
1491 DAG.getConstant(RoundWidth,
1492 TLI.getShiftAmountTy(Tmp3.getValueType())));
1493 Hi = DAG.getTruncStore(Tmp1, dl, Hi, Tmp2,
1494 ST->getPointerInfo().getWithOffset(IncrementSize),
1495 ExtraVT, isVolatile, isNonTemporal,
1496 MinAlign(Alignment, IncrementSize));
1498 // Big endian - avoid unaligned stores.
1499 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
1500 // Store the top RoundWidth bits.
1501 Hi = DAG.getNode(ISD::SRL, dl, Tmp3.getValueType(), Tmp3,
1502 DAG.getConstant(ExtraWidth,
1503 TLI.getShiftAmountTy(Tmp3.getValueType())));
1504 Hi = DAG.getTruncStore(Tmp1, dl, Hi, Tmp2, ST->getPointerInfo(),
1505 RoundVT, isVolatile, isNonTemporal, Alignment);
1507 // Store the remaining ExtraWidth bits.
1508 IncrementSize = RoundWidth / 8;
1509 Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
1510 DAG.getIntPtrConstant(IncrementSize));
1511 Lo = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2,
1512 ST->getPointerInfo().getWithOffset(IncrementSize),
1513 ExtraVT, isVolatile, isNonTemporal,
1514 MinAlign(Alignment, IncrementSize));
1517 // The order of the stores doesn't matter.
1518 Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
1520 if (Tmp1 != ST->getChain() || Tmp3 != ST->getValue() ||
1521 Tmp2 != ST->getBasePtr())
1522 Result = SDValue(DAG.UpdateNodeOperands(Result.getNode(),
1527 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
1528 default: assert(0 && "This action is not supported yet!");
1529 case TargetLowering::Legal:
1530 // If this is an unaligned store and the target doesn't support it,
1532 if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) {
1533 const Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
1534 unsigned ABIAlignment= TLI.getTargetData()->getABITypeAlignment(Ty);
1535 if (ST->getAlignment() < ABIAlignment)
1536 Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()),
1540 case TargetLowering::Custom:
1541 Result = TLI.LowerOperation(Result, DAG);
1544 // TRUNCSTORE:i16 i32 -> STORE i16
1545 assert(isTypeLegal(StVT) && "Do not know how to expand this store!");
1546 Tmp3 = DAG.getNode(ISD::TRUNCATE, dl, StVT, Tmp3);
1547 Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(),
1548 isVolatile, isNonTemporal, Alignment);
1556 assert(Result.getValueType() == Op.getValueType() &&
1557 "Bad legalization!");
1559 // Make sure that the generated code is itself legal.
1561 Result = LegalizeOp(Result);
1563 // Note that LegalizeOp may be reentered even from single-use nodes, which
1564 // means that we always must cache transformed nodes.
1565 AddLegalizedOperand(Op, Result);
1569 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1570 SDValue Vec = Op.getOperand(0);
1571 SDValue Idx = Op.getOperand(1);
1572 DebugLoc dl = Op.getDebugLoc();
1573 // Store the value to a temporary stack slot, then LOAD the returned part.
1574 SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType());
1575 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1576 MachinePointerInfo(), false, false, 0);
1578 // Add the offset to the index.
1580 Vec.getValueType().getVectorElementType().getSizeInBits()/8;
1581 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
1582 DAG.getConstant(EltSize, Idx.getValueType()));
1584 if (Idx.getValueType().bitsGT(TLI.getPointerTy()))
1585 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx);
1587 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx);
1589 StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr);
1591 if (Op.getValueType().isVector())
1592 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,MachinePointerInfo(),
1594 return DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1595 MachinePointerInfo(),
1596 Vec.getValueType().getVectorElementType(),
1600 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1601 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1603 SDValue Vec = Op.getOperand(0);
1604 SDValue Part = Op.getOperand(1);
1605 SDValue Idx = Op.getOperand(2);
1606 DebugLoc dl = Op.getDebugLoc();
1608 // Store the value to a temporary stack slot, then LOAD the returned part.
1610 SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType());
1611 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1612 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI);
1614 // First store the whole vector.
1615 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1618 // Then store the inserted part.
1620 // Add the offset to the index.
1622 Vec.getValueType().getVectorElementType().getSizeInBits()/8;
1624 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
1625 DAG.getConstant(EltSize, Idx.getValueType()));
1627 if (Idx.getValueType().bitsGT(TLI.getPointerTy()))
1628 Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx);
1630 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx);
1632 SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
1635 // Store the subvector.
1636 Ch = DAG.getStore(DAG.getEntryNode(), dl, Part, SubStackPtr,
1637 MachinePointerInfo(), false, false, 0);
1639 // Finally, load the updated vector.
1640 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo,
1644 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1645 // We can't handle this case efficiently. Allocate a sufficiently
1646 // aligned object on the stack, store each element into it, then load
1647 // the result as a vector.
1648 // Create the stack frame object.
1649 EVT VT = Node->getValueType(0);
1650 EVT EltVT = VT.getVectorElementType();
1651 DebugLoc dl = Node->getDebugLoc();
1652 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1653 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1654 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI);
1656 // Emit a store of each element to the stack slot.
1657 SmallVector<SDValue, 8> Stores;
1658 unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1659 // Store (in the right endianness) the elements to memory.
1660 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1661 // Ignore undef elements.
1662 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue;
1664 unsigned Offset = TypeByteSize*i;
1666 SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType());
1667 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1669 // If the destination vector element type is narrower than the source
1670 // element type, only store the bits necessary.
1671 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1672 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1673 Node->getOperand(i), Idx,
1674 PtrInfo.getWithOffset(Offset),
1675 EltVT, false, false, 0));
1677 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl,
1678 Node->getOperand(i), Idx,
1679 PtrInfo.getWithOffset(Offset),
1684 if (!Stores.empty()) // Not all undef elements?
1685 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1686 &Stores[0], Stores.size());
1688 StoreChain = DAG.getEntryNode();
1690 // Result is a load from the stack slot.
1691 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo, false, false, 0);
1694 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode* Node) {
1695 DebugLoc dl = Node->getDebugLoc();
1696 SDValue Tmp1 = Node->getOperand(0);
1697 SDValue Tmp2 = Node->getOperand(1);
1699 // Get the sign bit of the RHS. First obtain a value that has the same
1700 // sign as the sign bit, i.e. negative if and only if the sign bit is 1.
1702 EVT FloatVT = Tmp2.getValueType();
1703 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), FloatVT.getSizeInBits());
1704 if (isTypeLegal(IVT)) {
1705 // Convert to an integer with the same sign bit.
1706 SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2);
1708 // Store the float to memory, then load the sign part out as an integer.
1709 MVT LoadTy = TLI.getPointerTy();
1710 // First create a temporary that is aligned for both the load and store.
1711 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1712 // Then store the float to it.
1714 DAG.getStore(DAG.getEntryNode(), dl, Tmp2, StackPtr, MachinePointerInfo(),
1716 if (TLI.isBigEndian()) {
1717 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1718 // Load out a legal integer with the same sign bit as the float.
1719 SignBit = DAG.getLoad(LoadTy, dl, Ch, StackPtr, MachinePointerInfo(),
1721 } else { // Little endian
1722 SDValue LoadPtr = StackPtr;
1723 // The float may be wider than the integer we are going to load. Advance
1724 // the pointer so that the loaded integer will contain the sign bit.
1725 unsigned Strides = (FloatVT.getSizeInBits()-1)/LoadTy.getSizeInBits();
1726 unsigned ByteOffset = (Strides * LoadTy.getSizeInBits()) / 8;
1727 LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(),
1728 LoadPtr, DAG.getIntPtrConstant(ByteOffset));
1729 // Load a legal integer containing the sign bit.
1730 SignBit = DAG.getLoad(LoadTy, dl, Ch, LoadPtr, MachinePointerInfo(),
1732 // Move the sign bit to the top bit of the loaded integer.
1733 unsigned BitShift = LoadTy.getSizeInBits() -
1734 (FloatVT.getSizeInBits() - 8 * ByteOffset);
1735 assert(BitShift < LoadTy.getSizeInBits() && "Pointer advanced wrong?");
1737 SignBit = DAG.getNode(ISD::SHL, dl, LoadTy, SignBit,
1738 DAG.getConstant(BitShift,
1739 TLI.getShiftAmountTy(SignBit.getValueType())));
1742 // Now get the sign bit proper, by seeing whether the value is negative.
1743 SignBit = DAG.getSetCC(dl, TLI.getSetCCResultType(SignBit.getValueType()),
1744 SignBit, DAG.getConstant(0, SignBit.getValueType()),
1746 // Get the absolute value of the result.
1747 SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1);
1748 // Select between the nabs and abs value based on the sign bit of
1750 return DAG.getNode(ISD::SELECT, dl, AbsVal.getValueType(), SignBit,
1751 DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), AbsVal),
1755 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1756 SmallVectorImpl<SDValue> &Results) {
1757 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1758 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1759 " not tell us which reg is the stack pointer!");
1760 DebugLoc dl = Node->getDebugLoc();
1761 EVT VT = Node->getValueType(0);
1762 SDValue Tmp1 = SDValue(Node, 0);
1763 SDValue Tmp2 = SDValue(Node, 1);
1764 SDValue Tmp3 = Node->getOperand(2);
1765 SDValue Chain = Tmp1.getOperand(0);
1767 // Chain the dynamic stack allocation so that it doesn't modify the stack
1768 // pointer when other instructions are using the stack.
1769 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true));
1771 SDValue Size = Tmp2.getOperand(1);
1772 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1773 Chain = SP.getValue(1);
1774 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1775 unsigned StackAlign = TM.getFrameLowering()->getStackAlignment();
1776 if (Align > StackAlign)
1777 SP = DAG.getNode(ISD::AND, dl, VT, SP,
1778 DAG.getConstant(-(uint64_t)Align, VT));
1779 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1780 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1782 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true),
1783 DAG.getIntPtrConstant(0, true), SDValue());
1785 Results.push_back(Tmp1);
1786 Results.push_back(Tmp2);
1789 /// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and
1790 /// condition code CC on the current target. This routine expands SETCC with
1791 /// illegal condition code into AND / OR of multiple SETCC values.
1792 void SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT,
1793 SDValue &LHS, SDValue &RHS,
1796 EVT OpVT = LHS.getValueType();
1797 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1798 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1799 default: assert(0 && "Unknown condition code action!");
1800 case TargetLowering::Legal:
1803 case TargetLowering::Expand: {
1804 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1807 default: assert(0 && "Don't know how to expand this condition!");
1808 case ISD::SETOEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETO; Opc = ISD::AND; break;
1809 case ISD::SETOGT: CC1 = ISD::SETGT; CC2 = ISD::SETO; Opc = ISD::AND; break;
1810 case ISD::SETOGE: CC1 = ISD::SETGE; CC2 = ISD::SETO; Opc = ISD::AND; break;
1811 case ISD::SETOLT: CC1 = ISD::SETLT; CC2 = ISD::SETO; Opc = ISD::AND; break;
1812 case ISD::SETOLE: CC1 = ISD::SETLE; CC2 = ISD::SETO; Opc = ISD::AND; break;
1813 case ISD::SETONE: CC1 = ISD::SETNE; CC2 = ISD::SETO; Opc = ISD::AND; break;
1814 case ISD::SETUEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1815 case ISD::SETUGT: CC1 = ISD::SETGT; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1816 case ISD::SETUGE: CC1 = ISD::SETGE; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1817 case ISD::SETULT: CC1 = ISD::SETLT; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1818 case ISD::SETULE: CC1 = ISD::SETLE; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1819 case ISD::SETUNE: CC1 = ISD::SETNE; CC2 = ISD::SETUO; Opc = ISD::OR; break;
1820 // FIXME: Implement more expansions.
1823 SDValue SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1824 SDValue SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1825 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1833 /// EmitStackConvert - Emit a store/load combination to the stack. This stores
1834 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1835 /// a load from the stack slot to DestVT, extending it if needed.
1836 /// The resultant code need not be legal.
1837 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp,
1841 // Create the stack frame object.
1843 TLI.getTargetData()->getPrefTypeAlignment(SrcOp.getValueType().
1844 getTypeForEVT(*DAG.getContext()));
1845 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1847 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1848 int SPFI = StackPtrFI->getIndex();
1849 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI);
1851 unsigned SrcSize = SrcOp.getValueType().getSizeInBits();
1852 unsigned SlotSize = SlotVT.getSizeInBits();
1853 unsigned DestSize = DestVT.getSizeInBits();
1854 const Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1855 unsigned DestAlign = TLI.getTargetData()->getPrefTypeAlignment(DestType);
1857 // Emit a store to the stack slot. Use a truncstore if the input value is
1858 // later than DestVT.
1861 if (SrcSize > SlotSize)
1862 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr,
1863 PtrInfo, SlotVT, false, false, SrcAlign);
1865 assert(SrcSize == SlotSize && "Invalid store");
1866 Store = DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr,
1867 PtrInfo, false, false, SrcAlign);
1870 // Result is a load from the stack slot.
1871 if (SlotSize == DestSize)
1872 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo,
1873 false, false, DestAlign);
1875 assert(SlotSize < DestSize && "Unknown extension!");
1876 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr,
1877 PtrInfo, SlotVT, false, false, DestAlign);
1880 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1881 DebugLoc dl = Node->getDebugLoc();
1882 // Create a vector sized/aligned stack slot, store the value to element #0,
1883 // then load the whole vector back out.
1884 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1886 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1887 int SPFI = StackPtrFI->getIndex();
1889 SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), dl, Node->getOperand(0),
1891 MachinePointerInfo::getFixedStack(SPFI),
1892 Node->getValueType(0).getVectorElementType(),
1894 return DAG.getLoad(Node->getValueType(0), dl, Ch, StackPtr,
1895 MachinePointerInfo::getFixedStack(SPFI),
1900 /// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't
1901 /// support the operation, but do support the resultant vector type.
1902 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1903 unsigned NumElems = Node->getNumOperands();
1904 SDValue Value1, Value2;
1905 DebugLoc dl = Node->getDebugLoc();
1906 EVT VT = Node->getValueType(0);
1907 EVT OpVT = Node->getOperand(0).getValueType();
1908 EVT EltVT = VT.getVectorElementType();
1910 // If the only non-undef value is the low element, turn this into a
1911 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1912 bool isOnlyLowElement = true;
1913 bool MoreThanTwoValues = false;
1914 bool isConstant = true;
1915 for (unsigned i = 0; i < NumElems; ++i) {
1916 SDValue V = Node->getOperand(i);
1917 if (V.getOpcode() == ISD::UNDEF)
1920 isOnlyLowElement = false;
1921 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1924 if (!Value1.getNode()) {
1926 } else if (!Value2.getNode()) {
1929 } else if (V != Value1 && V != Value2) {
1930 MoreThanTwoValues = true;
1934 if (!Value1.getNode())
1935 return DAG.getUNDEF(VT);
1937 if (isOnlyLowElement)
1938 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1940 // If all elements are constants, create a load from the constant pool.
1942 std::vector<Constant*> CV;
1943 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1944 if (ConstantFPSDNode *V =
1945 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1946 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1947 } else if (ConstantSDNode *V =
1948 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1950 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1952 // If OpVT and EltVT don't match, EltVT is not legal and the
1953 // element values have been promoted/truncated earlier. Undo this;
1954 // we don't want a v16i8 to become a v16i32 for example.
1955 const ConstantInt *CI = V->getConstantIntValue();
1956 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1957 CI->getZExtValue()));
1960 assert(Node->getOperand(i).getOpcode() == ISD::UNDEF);
1961 const Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1962 CV.push_back(UndefValue::get(OpNTy));
1965 Constant *CP = ConstantVector::get(CV);
1966 SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy());
1967 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1968 return DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
1969 MachinePointerInfo::getConstantPool(),
1970 false, false, Alignment);
1973 if (!MoreThanTwoValues) {
1974 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1975 for (unsigned i = 0; i < NumElems; ++i) {
1976 SDValue V = Node->getOperand(i);
1977 if (V.getOpcode() == ISD::UNDEF)
1979 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1981 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1982 // Get the splatted value into the low element of a vector register.
1983 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1985 if (Value2.getNode())
1986 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1988 Vec2 = DAG.getUNDEF(VT);
1990 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1991 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec.data());
1995 // Otherwise, we can't handle this case efficiently.
1996 return ExpandVectorBuildThroughStack(Node);
1999 // ExpandLibCall - Expand a node into a call to a libcall. If the result value
2000 // does not fit into a register, return the lo part and set the hi part to the
2001 // by-reg argument. If it does fit into a single register, return the result
2002 // and leave the Hi part unset.
2003 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2005 // The input chain to this libcall is the entry node of the function.
2006 // Legalizing the call will automatically add the previous call to the
2008 SDValue InChain = DAG.getEntryNode();
2010 TargetLowering::ArgListTy Args;
2011 TargetLowering::ArgListEntry Entry;
2012 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
2013 EVT ArgVT = Node->getOperand(i).getValueType();
2014 const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2015 Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy;
2016 Entry.isSExt = isSigned;
2017 Entry.isZExt = !isSigned;
2018 Args.push_back(Entry);
2020 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2021 TLI.getPointerTy());
2023 // Splice the libcall in wherever FindInputOutputChains tells us to.
2024 const Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2026 // isTailCall may be true since the callee does not reference caller stack
2027 // frame. Check if it's in the right position.
2028 bool isTailCall = isInTailCallPosition(DAG, Node, TLI);
2029 std::pair<SDValue, SDValue> CallInfo =
2030 TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false,
2031 0, TLI.getLibcallCallingConv(LC), isTailCall,
2032 /*isReturnValueUsed=*/true,
2033 Callee, Args, DAG, Node->getDebugLoc());
2035 if (!CallInfo.second.getNode())
2036 // It's a tailcall, return the chain (which is the DAG root).
2037 return DAG.getRoot();
2039 // Legalize the call sequence, starting with the chain. This will advance
2040 // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that
2041 // was added by LowerCallTo (guaranteeing proper serialization of calls).
2042 LegalizeOp(CallInfo.second);
2043 return CallInfo.first;
2046 // ExpandChainLibCall - Expand a node into a call to a libcall. Similar to
2047 // ExpandLibCall except that the first operand is the in-chain.
2048 std::pair<SDValue, SDValue>
2049 SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2052 SDValue InChain = Node->getOperand(0);
2054 TargetLowering::ArgListTy Args;
2055 TargetLowering::ArgListEntry Entry;
2056 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2057 EVT ArgVT = Node->getOperand(i).getValueType();
2058 const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2059 Entry.Node = Node->getOperand(i);
2061 Entry.isSExt = isSigned;
2062 Entry.isZExt = !isSigned;
2063 Args.push_back(Entry);
2065 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2066 TLI.getPointerTy());
2068 // Splice the libcall in wherever FindInputOutputChains tells us to.
2069 const Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2070 std::pair<SDValue, SDValue> CallInfo =
2071 TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false,
2072 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false,
2073 /*isReturnValueUsed=*/true,
2074 Callee, Args, DAG, Node->getDebugLoc());
2076 // Legalize the call sequence, starting with the chain. This will advance
2077 // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that
2078 // was added by LowerCallTo (guaranteeing proper serialization of calls).
2079 LegalizeOp(CallInfo.second);
2083 SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2084 RTLIB::Libcall Call_F32,
2085 RTLIB::Libcall Call_F64,
2086 RTLIB::Libcall Call_F80,
2087 RTLIB::Libcall Call_PPCF128) {
2089 switch (Node->getValueType(0).getSimpleVT().SimpleTy) {
2090 default: assert(0 && "Unexpected request for libcall!");
2091 case MVT::f32: LC = Call_F32; break;
2092 case MVT::f64: LC = Call_F64; break;
2093 case MVT::f80: LC = Call_F80; break;
2094 case MVT::ppcf128: LC = Call_PPCF128; break;
2096 return ExpandLibCall(LC, Node, false);
2099 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2100 RTLIB::Libcall Call_I8,
2101 RTLIB::Libcall Call_I16,
2102 RTLIB::Libcall Call_I32,
2103 RTLIB::Libcall Call_I64,
2104 RTLIB::Libcall Call_I128) {
2106 switch (Node->getValueType(0).getSimpleVT().SimpleTy) {
2107 default: assert(0 && "Unexpected request for libcall!");
2108 case MVT::i8: LC = Call_I8; break;
2109 case MVT::i16: LC = Call_I16; break;
2110 case MVT::i32: LC = Call_I32; break;
2111 case MVT::i64: LC = Call_I64; break;
2112 case MVT::i128: LC = Call_I128; break;
2114 return ExpandLibCall(LC, Node, isSigned);
2117 /// ExpandLegalINT_TO_FP - This function is responsible for legalizing a
2118 /// INT_TO_FP operation of the specified operand when the target requests that
2119 /// we expand it. At this point, we know that the result and operand types are
2120 /// legal for the target.
2121 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
2125 if (Op0.getValueType() == MVT::i32) {
2126 // simple 32-bit [signed|unsigned] integer to float/double expansion
2128 // Get the stack frame index of a 8 byte buffer.
2129 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2131 // word offset constant for Hi/Lo address computation
2132 SDValue WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy());
2133 // set up Hi and Lo (into buffer) address based on endian
2134 SDValue Hi = StackSlot;
2135 SDValue Lo = DAG.getNode(ISD::ADD, dl,
2136 TLI.getPointerTy(), StackSlot, WordOff);
2137 if (TLI.isLittleEndian())
2140 // if signed map to unsigned space
2143 // constant used to invert sign bit (signed to unsigned mapping)
2144 SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32);
2145 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2149 // store the lo of the constructed double - based on integer input
2150 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl,
2151 Op0Mapped, Lo, MachinePointerInfo(),
2153 // initial hi portion of constructed double
2154 SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32);
2155 // store the hi of the constructed double - biased exponent
2156 SDValue Store2 = DAG.getStore(Store1, dl, InitialHi, Hi,
2157 MachinePointerInfo(),
2159 // load the constructed double
2160 SDValue Load = DAG.getLoad(MVT::f64, dl, Store2, StackSlot,
2161 MachinePointerInfo(), false, false, 0);
2162 // FP constant to bias correct the final result
2163 SDValue Bias = DAG.getConstantFP(isSigned ?
2164 BitsToDouble(0x4330000080000000ULL) :
2165 BitsToDouble(0x4330000000000000ULL),
2167 // subtract the bias
2168 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2171 // handle final rounding
2172 if (DestVT == MVT::f64) {
2175 } else if (DestVT.bitsLT(MVT::f64)) {
2176 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
2177 DAG.getIntPtrConstant(0));
2178 } else if (DestVT.bitsGT(MVT::f64)) {
2179 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
2183 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
2184 // Code below here assumes !isSigned without checking again.
2186 // Implementation of unsigned i64 to f64 following the algorithm in
2187 // __floatundidf in compiler_rt. This implementation has the advantage
2188 // of performing rounding correctly, both in the default rounding mode
2189 // and in all alternate rounding modes.
2190 // TODO: Generalize this for use with other types.
2191 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
2193 DAG.getConstant(UINT64_C(0x4330000000000000), MVT::i64);
2194 SDValue TwoP84PlusTwoP52 =
2195 DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), MVT::f64);
2197 DAG.getConstant(UINT64_C(0x4530000000000000), MVT::i64);
2199 SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
2200 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
2201 DAG.getConstant(32, MVT::i64));
2202 SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
2203 SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
2204 SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
2205 SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
2206 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
2208 return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
2211 // Implementation of unsigned i64 to f32.
2212 // TODO: Generalize this for use with other types.
2213 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) {
2214 // For unsigned conversions, convert them to signed conversions using the
2215 // algorithm from the x86_64 __floatundidf in compiler_rt.
2217 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2219 SDValue ShiftConst =
2220 DAG.getConstant(1, TLI.getShiftAmountTy(Op0.getValueType()));
2221 SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2222 SDValue AndConst = DAG.getConstant(1, MVT::i64);
2223 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
2224 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2226 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2227 SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2229 // TODO: This really should be implemented using a branch rather than a
2230 // select. We happen to get lucky and machinesink does the right
2231 // thing most of the time. This would be a good candidate for a
2232 //pseudo-op, or, even better, for whole-function isel.
2233 SDValue SignBitTest = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64),
2234 Op0, DAG.getConstant(0, MVT::i64), ISD::SETLT);
2235 return DAG.getNode(ISD::SELECT, dl, MVT::f32, SignBitTest, Slow, Fast);
2238 // Otherwise, implement the fully general conversion.
2240 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2241 DAG.getConstant(UINT64_C(0xfffffffffffff800), MVT::i64));
2242 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2243 DAG.getConstant(UINT64_C(0x800), MVT::i64));
2244 SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2245 DAG.getConstant(UINT64_C(0x7ff), MVT::i64));
2246 SDValue Ne = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64),
2247 And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE);
2248 SDValue Sel = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ne, Or, Op0);
2249 SDValue Ge = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64),
2250 Op0, DAG.getConstant(UINT64_C(0x0020000000000000), MVT::i64),
2252 SDValue Sel2 = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ge, Sel, Op0);
2253 EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType());
2255 SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2256 DAG.getConstant(32, SHVT));
2257 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2258 SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2260 DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), MVT::f64);
2261 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2262 SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2263 SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2264 SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2265 return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2266 DAG.getIntPtrConstant(0));
2269 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2271 SDValue SignSet = DAG.getSetCC(dl, TLI.getSetCCResultType(Op0.getValueType()),
2272 Op0, DAG.getConstant(0, Op0.getValueType()),
2274 SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4);
2275 SDValue CstOffset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(),
2276 SignSet, Four, Zero);
2278 // If the sign bit of the integer is set, the large number will be treated
2279 // as a negative number. To counteract this, the dynamic code adds an
2280 // offset depending on the data type.
2282 switch (Op0.getValueType().getSimpleVT().SimpleTy) {
2283 default: assert(0 && "Unsupported integer type!");
2284 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2285 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2286 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2287 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2289 if (TLI.isLittleEndian()) FF <<= 32;
2290 Constant *FudgeFactor = ConstantInt::get(
2291 Type::getInt64Ty(*DAG.getContext()), FF);
2293 SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy());
2294 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2295 CPIdx = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), CPIdx, CstOffset);
2296 Alignment = std::min(Alignment, 4u);
2298 if (DestVT == MVT::f32)
2299 FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2300 MachinePointerInfo::getConstantPool(),
2301 false, false, Alignment);
2304 LegalizeOp(DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT,
2305 DAG.getEntryNode(), CPIdx,
2306 MachinePointerInfo::getConstantPool(),
2307 MVT::f32, false, false, Alignment));
2310 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2313 /// PromoteLegalINT_TO_FP - This function is responsible for legalizing a
2314 /// *INT_TO_FP operation of the specified operand when the target requests that
2315 /// we promote it. At this point, we know that the result and operand types are
2316 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2317 /// operation that takes a larger input.
2318 SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp,
2322 // First step, figure out the appropriate *INT_TO_FP operation to use.
2323 EVT NewInTy = LegalOp.getValueType();
2325 unsigned OpToUse = 0;
2327 // Scan for the appropriate larger type to use.
2329 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2330 assert(NewInTy.isInteger() && "Ran out of possibilities!");
2332 // If the target supports SINT_TO_FP of this type, use it.
2333 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2334 OpToUse = ISD::SINT_TO_FP;
2337 if (isSigned) continue;
2339 // If the target supports UINT_TO_FP of this type, use it.
2340 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2341 OpToUse = ISD::UINT_TO_FP;
2345 // Otherwise, try a larger type.
2348 // Okay, we found the operation and type to use. Zero extend our input to the
2349 // desired type then run the operation on it.
2350 return DAG.getNode(OpToUse, dl, DestVT,
2351 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2352 dl, NewInTy, LegalOp));
2355 /// PromoteLegalFP_TO_INT - This function is responsible for legalizing a
2356 /// FP_TO_*INT operation of the specified operand when the target requests that
2357 /// we promote it. At this point, we know that the result and operand types are
2358 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2359 /// operation that returns a larger result.
2360 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp,
2364 // First step, figure out the appropriate FP_TO*INT operation to use.
2365 EVT NewOutTy = DestVT;
2367 unsigned OpToUse = 0;
2369 // Scan for the appropriate larger type to use.
2371 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2372 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2374 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2375 OpToUse = ISD::FP_TO_SINT;
2379 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2380 OpToUse = ISD::FP_TO_UINT;
2384 // Otherwise, try a larger type.
2388 // Okay, we found the operation and type to use.
2389 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2391 // Truncate the result of the extended FP_TO_*INT operation to the desired
2393 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2396 /// ExpandBSWAP - Open code the operations for BSWAP of the specified operation.
2398 SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, DebugLoc dl) {
2399 EVT VT = Op.getValueType();
2400 EVT SHVT = TLI.getShiftAmountTy(VT);
2401 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2402 switch (VT.getSimpleVT().SimpleTy) {
2403 default: assert(0 && "Unhandled Expand type in BSWAP!");
2405 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
2406 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
2407 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2409 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT));
2410 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
2411 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
2412 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT));
2413 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT));
2414 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT));
2415 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2416 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2417 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2419 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT));
2420 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT));
2421 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT));
2422 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT));
2423 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT));
2424 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT));
2425 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT));
2426 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT));
2427 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT));
2428 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT));
2429 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT));
2430 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT));
2431 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT));
2432 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT));
2433 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2434 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2435 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2436 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2437 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2438 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2439 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2443 /// SplatByte - Distribute ByteVal over NumBits bits.
2444 // FIXME: Move this helper to a common place.
2445 static APInt SplatByte(unsigned NumBits, uint8_t ByteVal) {
2446 APInt Val = APInt(NumBits, ByteVal);
2448 for (unsigned i = NumBits; i > 8; i >>= 1) {
2449 Val = (Val << Shift) | Val;
2455 /// ExpandBitCount - Expand the specified bitcount instruction into operations.
2457 SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
2460 default: assert(0 && "Cannot expand this yet!");
2462 EVT VT = Op.getValueType();
2463 EVT ShVT = TLI.getShiftAmountTy(VT);
2464 unsigned Len = VT.getSizeInBits();
2466 assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&
2467 "CTPOP not implemented for this type.");
2469 // This is the "best" algorithm from
2470 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
2472 SDValue Mask55 = DAG.getConstant(SplatByte(Len, 0x55), VT);
2473 SDValue Mask33 = DAG.getConstant(SplatByte(Len, 0x33), VT);
2474 SDValue Mask0F = DAG.getConstant(SplatByte(Len, 0x0F), VT);
2475 SDValue Mask01 = DAG.getConstant(SplatByte(Len, 0x01), VT);
2477 // v = v - ((v >> 1) & 0x55555555...)
2478 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2479 DAG.getNode(ISD::AND, dl, VT,
2480 DAG.getNode(ISD::SRL, dl, VT, Op,
2481 DAG.getConstant(1, ShVT)),
2483 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
2484 Op = DAG.getNode(ISD::ADD, dl, VT,
2485 DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2486 DAG.getNode(ISD::AND, dl, VT,
2487 DAG.getNode(ISD::SRL, dl, VT, Op,
2488 DAG.getConstant(2, ShVT)),
2490 // v = (v + (v >> 4)) & 0x0F0F0F0F...
2491 Op = DAG.getNode(ISD::AND, dl, VT,
2492 DAG.getNode(ISD::ADD, dl, VT, Op,
2493 DAG.getNode(ISD::SRL, dl, VT, Op,
2494 DAG.getConstant(4, ShVT))),
2496 // v = (v * 0x01010101...) >> (Len - 8)
2497 Op = DAG.getNode(ISD::SRL, dl, VT,
2498 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2499 DAG.getConstant(Len - 8, ShVT));
2504 // for now, we do this:
2505 // x = x | (x >> 1);
2506 // x = x | (x >> 2);
2508 // x = x | (x >>16);
2509 // x = x | (x >>32); // for 64-bit input
2510 // return popcount(~x);
2512 // but see also: http://www.hackersdelight.org/HDcode/nlz.cc
2513 EVT VT = Op.getValueType();
2514 EVT ShVT = TLI.getShiftAmountTy(VT);
2515 unsigned len = VT.getSizeInBits();
2516 for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
2517 SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT);
2518 Op = DAG.getNode(ISD::OR, dl, VT, Op,
2519 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2521 Op = DAG.getNOT(dl, Op, VT);
2522 return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2525 // for now, we use: { return popcount(~x & (x - 1)); }
2526 // unless the target has ctlz but not ctpop, in which case we use:
2527 // { return 32 - nlz(~x & (x-1)); }
2528 // see also http://www.hackersdelight.org/HDcode/ntz.cc
2529 EVT VT = Op.getValueType();
2530 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2531 DAG.getNOT(dl, Op, VT),
2532 DAG.getNode(ISD::SUB, dl, VT, Op,
2533 DAG.getConstant(1, VT)));
2534 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2535 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
2536 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
2537 return DAG.getNode(ISD::SUB, dl, VT,
2538 DAG.getConstant(VT.getSizeInBits(), VT),
2539 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2540 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2545 std::pair <SDValue, SDValue> SelectionDAGLegalize::ExpandAtomic(SDNode *Node) {
2546 unsigned Opc = Node->getOpcode();
2547 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2552 llvm_unreachable("Unhandled atomic intrinsic Expand!");
2554 case ISD::ATOMIC_SWAP:
2555 switch (VT.SimpleTy) {
2556 default: llvm_unreachable("Unexpected value type for atomic!");
2557 case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break;
2558 case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break;
2559 case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break;
2560 case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break;
2563 case ISD::ATOMIC_CMP_SWAP:
2564 switch (VT.SimpleTy) {
2565 default: llvm_unreachable("Unexpected value type for atomic!");
2566 case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break;
2567 case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break;
2568 case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break;
2569 case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break;
2572 case ISD::ATOMIC_LOAD_ADD:
2573 switch (VT.SimpleTy) {
2574 default: llvm_unreachable("Unexpected value type for atomic!");
2575 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break;
2576 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break;
2577 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break;
2578 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break;
2581 case ISD::ATOMIC_LOAD_SUB:
2582 switch (VT.SimpleTy) {
2583 default: llvm_unreachable("Unexpected value type for atomic!");
2584 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break;
2585 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break;
2586 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break;
2587 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break;
2590 case ISD::ATOMIC_LOAD_AND:
2591 switch (VT.SimpleTy) {
2592 default: llvm_unreachable("Unexpected value type for atomic!");
2593 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break;
2594 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break;
2595 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break;
2596 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break;
2599 case ISD::ATOMIC_LOAD_OR:
2600 switch (VT.SimpleTy) {
2601 default: llvm_unreachable("Unexpected value type for atomic!");
2602 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break;
2603 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break;
2604 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break;
2605 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break;
2608 case ISD::ATOMIC_LOAD_XOR:
2609 switch (VT.SimpleTy) {
2610 default: llvm_unreachable("Unexpected value type for atomic!");
2611 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break;
2612 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break;
2613 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break;
2614 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break;
2617 case ISD::ATOMIC_LOAD_NAND:
2618 switch (VT.SimpleTy) {
2619 default: llvm_unreachable("Unexpected value type for atomic!");
2620 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break;
2621 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break;
2622 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break;
2623 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break;
2628 return ExpandChainLibCall(LC, Node, false);
2631 void SelectionDAGLegalize::ExpandNode(SDNode *Node,
2632 SmallVectorImpl<SDValue> &Results) {
2633 DebugLoc dl = Node->getDebugLoc();
2634 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2635 switch (Node->getOpcode()) {
2639 Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
2640 Results.push_back(Tmp1);
2643 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2645 case ISD::FRAMEADDR:
2646 case ISD::RETURNADDR:
2647 case ISD::FRAME_TO_ARGS_OFFSET:
2648 Results.push_back(DAG.getConstant(0, Node->getValueType(0)));
2650 case ISD::FLT_ROUNDS_:
2651 Results.push_back(DAG.getConstant(1, Node->getValueType(0)));
2653 case ISD::EH_RETURN:
2657 case ISD::EH_SJLJ_LONGJMP:
2658 case ISD::EH_SJLJ_DISPATCHSETUP:
2659 // If the target didn't expand these, there's nothing to do, so just
2660 // preserve the chain and be done.
2661 Results.push_back(Node->getOperand(0));
2663 case ISD::EH_SJLJ_SETJMP:
2664 // If the target didn't expand this, just return 'zero' and preserve the
2666 Results.push_back(DAG.getConstant(0, MVT::i32));
2667 Results.push_back(Node->getOperand(0));
2669 case ISD::MEMBARRIER: {
2670 // If the target didn't lower this, lower it to '__sync_synchronize()' call
2671 TargetLowering::ArgListTy Args;
2672 std::pair<SDValue, SDValue> CallResult =
2673 TLI.LowerCallTo(Node->getOperand(0), Type::getVoidTy(*DAG.getContext()),
2674 false, false, false, false, 0, CallingConv::C,
2675 /*isTailCall=*/false,
2676 /*isReturnValueUsed=*/true,
2677 DAG.getExternalSymbol("__sync_synchronize",
2678 TLI.getPointerTy()),
2680 Results.push_back(CallResult.second);
2683 // By default, atomic intrinsics are marked Legal and lowered. Targets
2684 // which don't support them directly, however, may want libcalls, in which
2685 // case they mark them Expand, and we get here.
2686 case ISD::ATOMIC_SWAP:
2687 case ISD::ATOMIC_LOAD_ADD:
2688 case ISD::ATOMIC_LOAD_SUB:
2689 case ISD::ATOMIC_LOAD_AND:
2690 case ISD::ATOMIC_LOAD_OR:
2691 case ISD::ATOMIC_LOAD_XOR:
2692 case ISD::ATOMIC_LOAD_NAND:
2693 case ISD::ATOMIC_LOAD_MIN:
2694 case ISD::ATOMIC_LOAD_MAX:
2695 case ISD::ATOMIC_LOAD_UMIN:
2696 case ISD::ATOMIC_LOAD_UMAX:
2697 case ISD::ATOMIC_CMP_SWAP: {
2698 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(Node);
2699 Results.push_back(Tmp.first);
2700 Results.push_back(Tmp.second);
2703 case ISD::DYNAMIC_STACKALLOC:
2704 ExpandDYNAMIC_STACKALLOC(Node, Results);
2706 case ISD::MERGE_VALUES:
2707 for (unsigned i = 0; i < Node->getNumValues(); i++)
2708 Results.push_back(Node->getOperand(i));
2711 EVT VT = Node->getValueType(0);
2713 Results.push_back(DAG.getConstant(0, VT));
2715 assert(VT.isFloatingPoint() && "Unknown value type!");
2716 Results.push_back(DAG.getConstantFP(0, VT));
2721 // If this operation is not supported, lower it to 'abort()' call
2722 TargetLowering::ArgListTy Args;
2723 std::pair<SDValue, SDValue> CallResult =
2724 TLI.LowerCallTo(Node->getOperand(0), Type::getVoidTy(*DAG.getContext()),
2725 false, false, false, false, 0, CallingConv::C,
2726 /*isTailCall=*/false,
2727 /*isReturnValueUsed=*/true,
2728 DAG.getExternalSymbol("abort", TLI.getPointerTy()),
2730 Results.push_back(CallResult.second);
2735 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2736 Node->getValueType(0), dl);
2737 Results.push_back(Tmp1);
2739 case ISD::FP_EXTEND:
2740 Tmp1 = EmitStackConvert(Node->getOperand(0),
2741 Node->getOperand(0).getValueType(),
2742 Node->getValueType(0), dl);
2743 Results.push_back(Tmp1);
2745 case ISD::SIGN_EXTEND_INREG: {
2746 // NOTE: we could fall back on load/store here too for targets without
2747 // SAR. However, it is doubtful that any exist.
2748 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2749 EVT VT = Node->getValueType(0);
2750 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT);
2753 unsigned BitsDiff = VT.getScalarType().getSizeInBits() -
2754 ExtraVT.getScalarType().getSizeInBits();
2755 SDValue ShiftCst = DAG.getConstant(BitsDiff, ShiftAmountTy);
2756 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2757 Node->getOperand(0), ShiftCst);
2758 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2759 Results.push_back(Tmp1);
2762 case ISD::FP_ROUND_INREG: {
2763 // The only way we can lower this is to turn it into a TRUNCSTORE,
2764 // EXTLOAD pair, targetting a temporary location (a stack slot).
2766 // NOTE: there is a choice here between constantly creating new stack
2767 // slots and always reusing the same one. We currently always create
2768 // new ones, as reuse may inhibit scheduling.
2769 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2770 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
2771 Node->getValueType(0), dl);
2772 Results.push_back(Tmp1);
2775 case ISD::SINT_TO_FP:
2776 case ISD::UINT_TO_FP:
2777 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
2778 Node->getOperand(0), Node->getValueType(0), dl);
2779 Results.push_back(Tmp1);
2781 case ISD::FP_TO_UINT: {
2782 SDValue True, False;
2783 EVT VT = Node->getOperand(0).getValueType();
2784 EVT NVT = Node->getValueType(0);
2785 APFloat apf(APInt::getNullValue(VT.getSizeInBits()));
2786 APInt x = APInt::getSignBit(NVT.getSizeInBits());
2787 (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
2788 Tmp1 = DAG.getConstantFP(apf, VT);
2789 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(VT),
2790 Node->getOperand(0),
2792 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
2793 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
2794 DAG.getNode(ISD::FSUB, dl, VT,
2795 Node->getOperand(0), Tmp1));
2796 False = DAG.getNode(ISD::XOR, dl, NVT, False,
2797 DAG.getConstant(x, NVT));
2798 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, True, False);
2799 Results.push_back(Tmp1);
2803 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2804 EVT VT = Node->getValueType(0);
2805 Tmp1 = Node->getOperand(0);
2806 Tmp2 = Node->getOperand(1);
2807 unsigned Align = Node->getConstantOperandVal(3);
2809 SDValue VAListLoad = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2,
2810 MachinePointerInfo(V), false, false, 0);
2811 SDValue VAList = VAListLoad;
2813 if (Align > TLI.getMinStackArgumentAlignment()) {
2814 assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
2816 VAList = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList,
2817 DAG.getConstant(Align - 1,
2818 TLI.getPointerTy()));
2820 VAList = DAG.getNode(ISD::AND, dl, TLI.getPointerTy(), VAList,
2821 DAG.getConstant(-(int64_t)Align,
2822 TLI.getPointerTy()));
2825 // Increment the pointer, VAList, to the next vaarg
2826 Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList,
2827 DAG.getConstant(TLI.getTargetData()->
2828 getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())),
2829 TLI.getPointerTy()));
2830 // Store the incremented VAList to the legalized pointer
2831 Tmp3 = DAG.getStore(VAListLoad.getValue(1), dl, Tmp3, Tmp2,
2832 MachinePointerInfo(V), false, false, 0);
2833 // Load the actual argument out of the pointer VAList
2834 Results.push_back(DAG.getLoad(VT, dl, Tmp3, VAList, MachinePointerInfo(),
2836 Results.push_back(Results[0].getValue(1));
2840 // This defaults to loading a pointer from the input and storing it to the
2841 // output, returning the chain.
2842 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2843 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2844 Tmp1 = DAG.getLoad(TLI.getPointerTy(), dl, Node->getOperand(0),
2845 Node->getOperand(2), MachinePointerInfo(VS),
2847 Tmp1 = DAG.getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
2848 MachinePointerInfo(VD), false, false, 0);
2849 Results.push_back(Tmp1);
2852 case ISD::EXTRACT_VECTOR_ELT:
2853 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2854 // This must be an access of the only element. Return it.
2855 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2856 Node->getOperand(0));
2858 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
2859 Results.push_back(Tmp1);
2861 case ISD::EXTRACT_SUBVECTOR:
2862 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
2864 case ISD::INSERT_SUBVECTOR:
2865 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
2867 case ISD::CONCAT_VECTORS: {
2868 Results.push_back(ExpandVectorBuildThroughStack(Node));
2871 case ISD::SCALAR_TO_VECTOR:
2872 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
2874 case ISD::INSERT_VECTOR_ELT:
2875 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
2876 Node->getOperand(1),
2877 Node->getOperand(2), dl));
2879 case ISD::VECTOR_SHUFFLE: {
2880 SmallVector<int, 8> Mask;
2881 cast<ShuffleVectorSDNode>(Node)->getMask(Mask);
2883 EVT VT = Node->getValueType(0);
2884 EVT EltVT = VT.getVectorElementType();
2885 if (getTypeAction(EltVT) == Promote)
2886 EltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
2887 unsigned NumElems = VT.getVectorNumElements();
2888 SmallVector<SDValue, 8> Ops;
2889 for (unsigned i = 0; i != NumElems; ++i) {
2891 Ops.push_back(DAG.getUNDEF(EltVT));
2894 unsigned Idx = Mask[i];
2896 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
2897 Node->getOperand(0),
2898 DAG.getIntPtrConstant(Idx)));
2900 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
2901 Node->getOperand(1),
2902 DAG.getIntPtrConstant(Idx - NumElems)));
2904 Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size());
2905 Results.push_back(Tmp1);
2908 case ISD::EXTRACT_ELEMENT: {
2909 EVT OpTy = Node->getOperand(0).getValueType();
2910 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
2912 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
2913 DAG.getConstant(OpTy.getSizeInBits()/2,
2914 TLI.getShiftAmountTy(Node->getOperand(0).getValueType())));
2915 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
2918 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
2919 Node->getOperand(0));
2921 Results.push_back(Tmp1);
2924 case ISD::STACKSAVE:
2925 // Expand to CopyFromReg if the target set
2926 // StackPointerRegisterToSaveRestore.
2927 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
2928 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
2929 Node->getValueType(0)));
2930 Results.push_back(Results[0].getValue(1));
2932 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
2933 Results.push_back(Node->getOperand(0));
2936 case ISD::STACKRESTORE:
2937 // Expand to CopyToReg if the target set
2938 // StackPointerRegisterToSaveRestore.
2939 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
2940 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
2941 Node->getOperand(1)));
2943 Results.push_back(Node->getOperand(0));
2946 case ISD::FCOPYSIGN:
2947 Results.push_back(ExpandFCOPYSIGN(Node));
2950 // Expand Y = FNEG(X) -> Y = SUB -0.0, X
2951 Tmp1 = DAG.getConstantFP(-0.0, Node->getValueType(0));
2952 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
2953 Node->getOperand(0));
2954 Results.push_back(Tmp1);
2957 // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X).
2958 EVT VT = Node->getValueType(0);
2959 Tmp1 = Node->getOperand(0);
2960 Tmp2 = DAG.getConstantFP(0.0, VT);
2961 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(Tmp1.getValueType()),
2962 Tmp1, Tmp2, ISD::SETUGT);
2963 Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1);
2964 Tmp1 = DAG.getNode(ISD::SELECT, dl, VT, Tmp2, Tmp1, Tmp3);
2965 Results.push_back(Tmp1);
2969 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
2970 RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128));
2973 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
2974 RTLIB::SIN_F80, RTLIB::SIN_PPCF128));
2977 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
2978 RTLIB::COS_F80, RTLIB::COS_PPCF128));
2981 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
2982 RTLIB::LOG_F80, RTLIB::LOG_PPCF128));
2985 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
2986 RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128));
2989 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
2990 RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128));
2993 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
2994 RTLIB::EXP_F80, RTLIB::EXP_PPCF128));
2997 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
2998 RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128));
3001 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
3002 RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128));
3005 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
3006 RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128));
3009 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
3010 RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128));
3013 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
3014 RTLIB::RINT_F80, RTLIB::RINT_PPCF128));
3016 case ISD::FNEARBYINT:
3017 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
3018 RTLIB::NEARBYINT_F64,
3019 RTLIB::NEARBYINT_F80,
3020 RTLIB::NEARBYINT_PPCF128));
3023 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
3024 RTLIB::POWI_F80, RTLIB::POWI_PPCF128));
3027 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
3028 RTLIB::POW_F80, RTLIB::POW_PPCF128));
3031 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
3032 RTLIB::DIV_F80, RTLIB::DIV_PPCF128));
3035 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
3036 RTLIB::REM_F80, RTLIB::REM_PPCF128));
3038 case ISD::FP16_TO_FP32:
3039 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
3041 case ISD::FP32_TO_FP16:
3042 Results.push_back(ExpandLibCall(RTLIB::FPROUND_F32_F16, Node, false));
3044 case ISD::ConstantFP: {
3045 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3046 // Check to see if this FP immediate is already legal.
3047 // If this is a legal constant, turn it into a TargetConstantFP node.
3048 if (TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3049 Results.push_back(SDValue(Node, 0));
3051 Results.push_back(ExpandConstantFP(CFP, true, DAG, TLI));
3054 case ISD::EHSELECTION: {
3055 unsigned Reg = TLI.getExceptionSelectorRegister();
3056 assert(Reg && "Can't expand to unknown register!");
3057 Results.push_back(DAG.getCopyFromReg(Node->getOperand(1), dl, Reg,
3058 Node->getValueType(0)));
3059 Results.push_back(Results[0].getValue(1));
3062 case ISD::EXCEPTIONADDR: {
3063 unsigned Reg = TLI.getExceptionAddressRegister();
3064 assert(Reg && "Can't expand to unknown register!");
3065 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, Reg,
3066 Node->getValueType(0)));
3067 Results.push_back(Results[0].getValue(1));
3071 EVT VT = Node->getValueType(0);
3072 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3073 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3074 "Don't know how to expand this subtraction!");
3075 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3076 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT));
3077 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp2, DAG.getConstant(1, VT));
3078 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3083 EVT VT = Node->getValueType(0);
3084 SDVTList VTs = DAG.getVTList(VT, VT);
3085 bool isSigned = Node->getOpcode() == ISD::SREM;
3086 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3087 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3088 Tmp2 = Node->getOperand(0);
3089 Tmp3 = Node->getOperand(1);
3090 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3091 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3092 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3094 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3095 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3096 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3097 } else if (isSigned) {
3098 Tmp1 = ExpandIntLibCall(Node, true,
3100 RTLIB::SREM_I16, RTLIB::SREM_I32,
3101 RTLIB::SREM_I64, RTLIB::SREM_I128);
3103 Tmp1 = ExpandIntLibCall(Node, false,
3105 RTLIB::UREM_I16, RTLIB::UREM_I32,
3106 RTLIB::UREM_I64, RTLIB::UREM_I128);
3108 Results.push_back(Tmp1);
3113 bool isSigned = Node->getOpcode() == ISD::SDIV;
3114 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3115 EVT VT = Node->getValueType(0);
3116 SDVTList VTs = DAG.getVTList(VT, VT);
3117 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT))
3118 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3119 Node->getOperand(1));
3121 Tmp1 = ExpandIntLibCall(Node, true,
3123 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
3124 RTLIB::SDIV_I64, RTLIB::SDIV_I128);
3126 Tmp1 = ExpandIntLibCall(Node, false,
3128 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
3129 RTLIB::UDIV_I64, RTLIB::UDIV_I128);
3130 Results.push_back(Tmp1);
3135 unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI :
3137 EVT VT = Node->getValueType(0);
3138 SDVTList VTs = DAG.getVTList(VT, VT);
3139 assert(TLI.isOperationLegalOrCustom(ExpandOpcode, VT) &&
3140 "If this wasn't legal, it shouldn't have been created!");
3141 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3142 Node->getOperand(1));
3143 Results.push_back(Tmp1.getValue(1));
3147 EVT VT = Node->getValueType(0);
3148 SDVTList VTs = DAG.getVTList(VT, VT);
3149 // See if multiply or divide can be lowered using two-result operations.
3150 // We just need the low half of the multiply; try both the signed
3151 // and unsigned forms. If the target supports both SMUL_LOHI and
3152 // UMUL_LOHI, form a preference by checking which forms of plain
3153 // MULH it supports.
3154 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3155 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3156 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3157 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3158 unsigned OpToUse = 0;
3159 if (HasSMUL_LOHI && !HasMULHS) {
3160 OpToUse = ISD::SMUL_LOHI;
3161 } else if (HasUMUL_LOHI && !HasMULHU) {
3162 OpToUse = ISD::UMUL_LOHI;
3163 } else if (HasSMUL_LOHI) {
3164 OpToUse = ISD::SMUL_LOHI;
3165 } else if (HasUMUL_LOHI) {
3166 OpToUse = ISD::UMUL_LOHI;
3169 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3170 Node->getOperand(1)));
3173 Tmp1 = ExpandIntLibCall(Node, false,
3175 RTLIB::MUL_I16, RTLIB::MUL_I32,
3176 RTLIB::MUL_I64, RTLIB::MUL_I128);
3177 Results.push_back(Tmp1);
3182 SDValue LHS = Node->getOperand(0);
3183 SDValue RHS = Node->getOperand(1);
3184 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3185 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3187 Results.push_back(Sum);
3188 EVT OType = Node->getValueType(1);
3190 SDValue Zero = DAG.getConstant(0, LHS.getValueType());
3192 // LHSSign -> LHS >= 0
3193 // RHSSign -> RHS >= 0
3194 // SumSign -> Sum >= 0
3197 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3199 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3201 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3202 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3203 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3204 Node->getOpcode() == ISD::SADDO ?
3205 ISD::SETEQ : ISD::SETNE);
3207 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3208 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3210 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3211 Results.push_back(Cmp);
3216 SDValue LHS = Node->getOperand(0);
3217 SDValue RHS = Node->getOperand(1);
3218 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
3219 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3221 Results.push_back(Sum);
3222 Results.push_back(DAG.getSetCC(dl, Node->getValueType(1), Sum, LHS,
3223 Node->getOpcode () == ISD::UADDO ?
3224 ISD::SETULT : ISD::SETUGT));
3229 EVT VT = Node->getValueType(0);
3230 SDValue LHS = Node->getOperand(0);
3231 SDValue RHS = Node->getOperand(1);
3234 static const unsigned Ops[2][3] =
3235 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3236 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3237 bool isSigned = Node->getOpcode() == ISD::SMULO;
3238 if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3239 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3240 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3241 } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3242 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3244 TopHalf = BottomHalf.getValue(1);
3245 } else if (TLI.isTypeLegal(EVT::getIntegerVT(*DAG.getContext(),
3246 VT.getSizeInBits() * 2))) {
3247 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3248 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3249 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3250 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3251 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3252 DAG.getIntPtrConstant(0));
3253 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3254 DAG.getIntPtrConstant(1));
3256 // We can fall back to a libcall with an illegal type for the MUL if we
3257 // have a libcall big enough.
3258 // Also, we can fall back to a division in some cases, but that's a big
3259 // performance hit in the general case.
3260 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3261 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3262 if (WideVT == MVT::i16)
3263 LC = RTLIB::MUL_I16;
3264 else if (WideVT == MVT::i32)
3265 LC = RTLIB::MUL_I32;
3266 else if (WideVT == MVT::i64)
3267 LC = RTLIB::MUL_I64;
3268 else if (WideVT == MVT::i128)
3269 LC = RTLIB::MUL_I128;
3270 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
3271 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3272 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3274 SDValue Ret = ExpandLibCall(LC, Node, isSigned);
3275 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Ret);
3276 TopHalf = DAG.getNode(ISD::SRL, dl, Ret.getValueType(), Ret,
3277 DAG.getConstant(VT.getSizeInBits(), TLI.getPointerTy()));
3278 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, TopHalf);
3281 Tmp1 = DAG.getConstant(VT.getSizeInBits() - 1,
3282 TLI.getShiftAmountTy(BottomHalf.getValueType()));
3283 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3284 TopHalf = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), TopHalf, Tmp1,
3287 TopHalf = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), TopHalf,
3288 DAG.getConstant(0, VT), ISD::SETNE);
3290 Results.push_back(BottomHalf);
3291 Results.push_back(TopHalf);
3294 case ISD::BUILD_PAIR: {
3295 EVT PairTy = Node->getValueType(0);
3296 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3297 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3298 Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2,
3299 DAG.getConstant(PairTy.getSizeInBits()/2,
3300 TLI.getShiftAmountTy(PairTy)));
3301 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3305 Tmp1 = Node->getOperand(0);
3306 Tmp2 = Node->getOperand(1);
3307 Tmp3 = Node->getOperand(2);
3308 if (Tmp1.getOpcode() == ISD::SETCC) {
3309 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3311 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3313 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3314 DAG.getConstant(0, Tmp1.getValueType()),
3315 Tmp2, Tmp3, ISD::SETNE);
3317 Results.push_back(Tmp1);
3320 SDValue Chain = Node->getOperand(0);
3321 SDValue Table = Node->getOperand(1);
3322 SDValue Index = Node->getOperand(2);
3324 EVT PTy = TLI.getPointerTy();
3326 const TargetData &TD = *TLI.getTargetData();
3327 unsigned EntrySize =
3328 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3330 Index = DAG.getNode(ISD::MUL, dl, PTy,
3331 Index, DAG.getConstant(EntrySize, PTy));
3332 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table);
3334 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3335 SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3336 MachinePointerInfo::getJumpTable(), MemVT,
3339 if (TM.getRelocationModel() == Reloc::PIC_) {
3340 // For PIC, the sequence is:
3341 // BRIND(load(Jumptable + index) + RelocBase)
3342 // RelocBase can be JumpTable, GOT or some sort of global base.
3343 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3344 TLI.getPICJumpTableRelocBase(Table, DAG));
3346 Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
3347 Results.push_back(Tmp1);
3351 // Expand brcond's setcc into its constituent parts and create a BR_CC
3353 Tmp1 = Node->getOperand(0);
3354 Tmp2 = Node->getOperand(1);
3355 if (Tmp2.getOpcode() == ISD::SETCC) {
3356 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3357 Tmp1, Tmp2.getOperand(2),
3358 Tmp2.getOperand(0), Tmp2.getOperand(1),
3359 Node->getOperand(2));
3361 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3362 DAG.getCondCode(ISD::SETNE), Tmp2,
3363 DAG.getConstant(0, Tmp2.getValueType()),
3364 Node->getOperand(2));
3366 Results.push_back(Tmp1);
3369 Tmp1 = Node->getOperand(0);
3370 Tmp2 = Node->getOperand(1);
3371 Tmp3 = Node->getOperand(2);
3372 LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, Tmp3, dl);
3374 // If we expanded the SETCC into an AND/OR, return the new node
3375 if (Tmp2.getNode() == 0) {
3376 Results.push_back(Tmp1);
3380 // Otherwise, SETCC for the given comparison type must be completely
3381 // illegal; expand it into a SELECT_CC.
3382 EVT VT = Node->getValueType(0);
3383 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3384 DAG.getConstant(1, VT), DAG.getConstant(0, VT), Tmp3);
3385 Results.push_back(Tmp1);
3388 case ISD::SELECT_CC: {
3389 Tmp1 = Node->getOperand(0); // LHS
3390 Tmp2 = Node->getOperand(1); // RHS
3391 Tmp3 = Node->getOperand(2); // True
3392 Tmp4 = Node->getOperand(3); // False
3393 SDValue CC = Node->getOperand(4);
3395 LegalizeSetCCCondCode(TLI.getSetCCResultType(Tmp1.getValueType()),
3396 Tmp1, Tmp2, CC, dl);
3398 assert(!Tmp2.getNode() && "Can't legalize SELECT_CC with legal condition!");
3399 Tmp2 = DAG.getConstant(0, Tmp1.getValueType());
3400 CC = DAG.getCondCode(ISD::SETNE);
3401 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, Tmp2,
3403 Results.push_back(Tmp1);
3407 Tmp1 = Node->getOperand(0); // Chain
3408 Tmp2 = Node->getOperand(2); // LHS
3409 Tmp3 = Node->getOperand(3); // RHS
3410 Tmp4 = Node->getOperand(1); // CC
3412 LegalizeSetCCCondCode(TLI.getSetCCResultType(Tmp2.getValueType()),
3413 Tmp2, Tmp3, Tmp4, dl);
3414 LastCALLSEQ_END = DAG.getEntryNode();
3416 assert(!Tmp3.getNode() && "Can't legalize BR_CC with legal condition!");
3417 Tmp3 = DAG.getConstant(0, Tmp2.getValueType());
3418 Tmp4 = DAG.getCondCode(ISD::SETNE);
3419 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, Tmp2,
3420 Tmp3, Node->getOperand(4));
3421 Results.push_back(Tmp1);
3424 case ISD::GLOBAL_OFFSET_TABLE:
3425 case ISD::GlobalAddress:
3426 case ISD::GlobalTLSAddress:
3427 case ISD::ExternalSymbol:
3428 case ISD::ConstantPool:
3429 case ISD::JumpTable:
3430 case ISD::INTRINSIC_W_CHAIN:
3431 case ISD::INTRINSIC_WO_CHAIN:
3432 case ISD::INTRINSIC_VOID:
3433 // FIXME: Custom lowering for these operations shouldn't return null!
3434 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
3435 Results.push_back(SDValue(Node, i));
3439 void SelectionDAGLegalize::PromoteNode(SDNode *Node,
3440 SmallVectorImpl<SDValue> &Results) {
3441 EVT OVT = Node->getValueType(0);
3442 if (Node->getOpcode() == ISD::UINT_TO_FP ||
3443 Node->getOpcode() == ISD::SINT_TO_FP ||
3444 Node->getOpcode() == ISD::SETCC) {
3445 OVT = Node->getOperand(0).getValueType();
3447 EVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
3448 DebugLoc dl = Node->getDebugLoc();
3449 SDValue Tmp1, Tmp2, Tmp3;
3450 switch (Node->getOpcode()) {
3454 // Zero extend the argument.
3455 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
3456 // Perform the larger operation.
3457 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
3458 if (Node->getOpcode() == ISD::CTTZ) {
3459 //if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT)
3460 Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT),
3461 Tmp1, DAG.getConstant(NVT.getSizeInBits(), NVT),
3463 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2,
3464 DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1);
3465 } else if (Node->getOpcode() == ISD::CTLZ) {
3466 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
3467 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
3468 DAG.getConstant(NVT.getSizeInBits() -
3469 OVT.getSizeInBits(), NVT));
3471 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
3474 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
3475 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
3476 Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1);
3477 Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1,
3478 DAG.getConstant(DiffBits, TLI.getShiftAmountTy(NVT)));
3479 Results.push_back(Tmp1);
3482 case ISD::FP_TO_UINT:
3483 case ISD::FP_TO_SINT:
3484 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
3485 Node->getOpcode() == ISD::FP_TO_SINT, dl);
3486 Results.push_back(Tmp1);
3488 case ISD::UINT_TO_FP:
3489 case ISD::SINT_TO_FP:
3490 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
3491 Node->getOpcode() == ISD::SINT_TO_FP, dl);
3492 Results.push_back(Tmp1);
3497 unsigned ExtOp, TruncOp;
3498 if (OVT.isVector()) {
3499 ExtOp = ISD::BITCAST;
3500 TruncOp = ISD::BITCAST;
3502 assert(OVT.isInteger() && "Cannot promote logic operation");
3503 ExtOp = ISD::ANY_EXTEND;
3504 TruncOp = ISD::TRUNCATE;
3506 // Promote each of the values to the new type.
3507 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
3508 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
3509 // Perform the larger operation, then convert back
3510 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
3511 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
3515 unsigned ExtOp, TruncOp;
3516 if (Node->getValueType(0).isVector()) {
3517 ExtOp = ISD::BITCAST;
3518 TruncOp = ISD::BITCAST;
3519 } else if (Node->getValueType(0).isInteger()) {
3520 ExtOp = ISD::ANY_EXTEND;
3521 TruncOp = ISD::TRUNCATE;
3523 ExtOp = ISD::FP_EXTEND;
3524 TruncOp = ISD::FP_ROUND;
3526 Tmp1 = Node->getOperand(0);
3527 // Promote each of the values to the new type.
3528 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
3529 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
3530 // Perform the larger operation, then round down.
3531 Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp1, Tmp2, Tmp3);
3532 if (TruncOp != ISD::FP_ROUND)
3533 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
3535 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
3536 DAG.getIntPtrConstant(0));
3537 Results.push_back(Tmp1);
3540 case ISD::VECTOR_SHUFFLE: {
3541 SmallVector<int, 8> Mask;
3542 cast<ShuffleVectorSDNode>(Node)->getMask(Mask);
3544 // Cast the two input vectors.
3545 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
3546 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
3548 // Convert the shuffle mask to the right # elements.
3549 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
3550 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
3551 Results.push_back(Tmp1);
3555 unsigned ExtOp = ISD::FP_EXTEND;
3556 if (NVT.isInteger()) {
3557 ISD::CondCode CCCode =
3558 cast<CondCodeSDNode>(Node->getOperand(2))->get();
3559 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3561 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
3562 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
3563 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3564 Tmp1, Tmp2, Node->getOperand(2)));
3570 // SelectionDAG::Legalize - This is the entry point for the file.
3572 void SelectionDAG::Legalize(CodeGenOpt::Level OptLevel) {
3573 /// run - This is the main entry point to this class.
3575 SelectionDAGLegalize(*this, OptLevel).LegalizeDAG();