X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAGISel.cpp;h=6299bc4b851f7270dc982993b8f561ab926a438a;hb=bd3ba461eb5578a81ba09ff7bd7eb271d1130196;hp=dc335f8ef783ff663f2bcc873e19b7bde5f3f939;hpb=b606dba13ba84721cc297174c2eb1275bc642604;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index dc335f8ef78..6299bc4b851 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -42,14 +42,35 @@ #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" -#include "llvm/Support/MathExtras.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/Timer.h" #include using namespace llvm; +static cl::opt +EnableValueProp("enable-value-prop", cl::Hidden); +static cl::opt +EnableLegalizeTypes("enable-legalize-types", cl::Hidden); + + #ifndef NDEBUG static cl::opt +ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, + cl::desc("Pop up a window to show dags before the first " + "dag combine pass")); +static cl::opt +ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, + cl::desc("Pop up a window to show dags before legalize types")); +static cl::opt +ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, + cl::desc("Pop up a window to show dags before legalize")); +static cl::opt +ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, + cl::desc("Pop up a window to show dags before the second " + "dag combine pass")); +static cl::opt ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected")); static cl::opt @@ -59,7 +80,11 @@ static cl::opt ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed")); #else -static const bool ViewISelDAGs = 0, ViewSchedDAGs = 0, ViewSUnitDAGs = 0; +static const bool ViewDAGCombine1 = false, + ViewLegalizeTypesDAGs = false, ViewLegalizeDAGs = false, + ViewDAGCombine2 = false, + ViewISelDAGs = false, ViewSchedDAGs = false, + ViewSUnitDAGs = false; #endif //===---------------------------------------------------------------------===// @@ -74,94 +99,148 @@ MachinePassRegistry RegisterScheduler::Registry; /// ISHeuristic command line option for instruction schedulers. /// //===---------------------------------------------------------------------===// -namespace { - cl::opt > - ISHeuristic("pre-RA-sched", - cl::init(&createDefaultScheduler), - cl::desc("Instruction schedulers available (before register" - " allocation):")); - - static RegisterScheduler - defaultListDAGScheduler("default", " Best scheduler for the target", - createDefaultScheduler); -} // namespace +static cl::opt > +ISHeuristic("pre-RA-sched", + cl::init(&createDefaultScheduler), + cl::desc("Instruction schedulers available (before register" + " allocation):")); + +static RegisterScheduler +defaultListDAGScheduler("default", " Best scheduler for the target", + createDefaultScheduler); namespace { struct SDISelAsmOperandInfo; } +/// ComputeLinearIndex - Given an LLVM IR aggregate type and a sequence +/// insertvalue or extractvalue indices that identify a member, return +/// the linearized index of the start of the member. +/// +static unsigned ComputeLinearIndex(const TargetLowering &TLI, const Type *Ty, + const unsigned *Indices, + const unsigned *IndicesEnd, + unsigned CurIndex = 0) { + // Base case: We're done. + if (Indices && Indices == IndicesEnd) + return CurIndex; + + // Given a struct type, recursively traverse the elements. + if (const StructType *STy = dyn_cast(Ty)) { + for (StructType::element_iterator EB = STy->element_begin(), + EI = EB, + EE = STy->element_end(); + EI != EE; ++EI) { + if (Indices && *Indices == unsigned(EI - EB)) + return ComputeLinearIndex(TLI, *EI, Indices+1, IndicesEnd, CurIndex); + CurIndex = ComputeLinearIndex(TLI, *EI, 0, 0, CurIndex); + } + } + // Given an array type, recursively traverse the elements. + else if (const ArrayType *ATy = dyn_cast(Ty)) { + const Type *EltTy = ATy->getElementType(); + for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) { + if (Indices && *Indices == i) + return ComputeLinearIndex(TLI, EltTy, Indices+1, IndicesEnd, CurIndex); + CurIndex = ComputeLinearIndex(TLI, EltTy, 0, 0, CurIndex); + } + } + // We haven't found the type we're looking for, so keep searching. + return CurIndex + 1; +} + /// ComputeValueVTs - Given an LLVM IR type, compute a sequence of -/// MVT::ValueTypes that represent all the individual underlying +/// MVTs that represent all the individual underlying /// non-aggregate types that comprise it. +/// +/// If Offsets is non-null, it points to a vector to be filled in +/// with the in-memory offsets of each of the individual values. +/// static void ComputeValueVTs(const TargetLowering &TLI, const Type *Ty, - SmallVectorImpl &ValueVTs) { + SmallVectorImpl &ValueVTs, + SmallVectorImpl *Offsets = 0, + uint64_t StartingOffset = 0) { // Given a struct type, recursively traverse the elements. if (const StructType *STy = dyn_cast(Ty)) { - for (StructType::element_iterator EI = STy->element_begin(), - EB = STy->element_end(); - EI != EB; ++EI) - ComputeValueVTs(TLI, *EI, ValueVTs); + const StructLayout *SL = TLI.getTargetData()->getStructLayout(STy); + for (StructType::element_iterator EB = STy->element_begin(), + EI = EB, + EE = STy->element_end(); + EI != EE; ++EI) + ComputeValueVTs(TLI, *EI, ValueVTs, Offsets, + StartingOffset + SL->getElementOffset(EI - EB)); return; } // Given an array type, recursively traverse the elements. if (const ArrayType *ATy = dyn_cast(Ty)) { const Type *EltTy = ATy->getElementType(); + uint64_t EltSize = TLI.getTargetData()->getABITypeSize(EltTy); for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) - ComputeValueVTs(TLI, EltTy, ValueVTs); + ComputeValueVTs(TLI, EltTy, ValueVTs, Offsets, + StartingOffset + i * EltSize); return; } - // Base case: we can get an MVT::ValueType for this LLVM IR type. + // Base case: we can get an MVT for this LLVM IR type. ValueVTs.push_back(TLI.getValueType(Ty)); + if (Offsets) + Offsets->push_back(StartingOffset); } namespace { - /// RegsForValue - This struct represents the physical registers that a - /// particular value is assigned and the type information about the value. - /// This is needed because values can be promoted into larger registers and - /// expanded into multiple smaller registers than the value. + /// RegsForValue - This struct represents the registers (physical or virtual) + /// that a particular set of values is assigned, and the type information about + /// the value. The most common situation is to represent one value at a time, + /// but struct or array values are handled element-wise as multiple values. + /// The splitting of aggregates is performed recursively, so that we never + /// have aggregate-typed registers. The values at this point do not necessarily + /// have legal types, so each value may require one or more registers of some + /// legal type. + /// struct VISIBILITY_HIDDEN RegsForValue { /// TLI - The TargetLowering object. + /// const TargetLowering *TLI; - /// Regs - This list holds the register (for legal and promoted values) - /// or register set (for expanded values) that the value should be assigned - /// to. - SmallVector Regs; + /// ValueVTs - The value types of the values, which may not be legal, and + /// may need be promoted or synthesized from one or more registers. + /// + SmallVector ValueVTs; - /// RegVTs - The value types of the registers. This is the same size - /// as ValueVTs; every register contributing to a given value must - /// have the same type. When Regs contains all virtual registers, the - /// contents of RegVTs is redundant with TLI's getRegisterType member - /// function, however when Regs contains physical registers, it is - /// necessary to have a separate record of the types. + /// RegVTs - The value types of the registers. This is the same size as + /// ValueVTs and it records, for each value, what the type of the assigned + /// register or registers are. (Individual values are never synthesized + /// from more than one type of register.) + /// + /// With virtual registers, the contents of RegVTs is redundant with TLI's + /// getRegisterType member function, however when with physical registers + /// it is necessary to have a separate record of the types. /// - SmallVector RegVTs; + SmallVector RegVTs; - /// ValueVTs - The value types of the values, which may be promoted - /// or synthesized from one or more registers. - SmallVector ValueVTs; + /// Regs - This list holds the registers assigned to the values. + /// Each legal or promoted value requires one register, and each + /// expanded value requires multiple registers. + /// + SmallVector Regs; RegsForValue() : TLI(0) {} - RegsForValue(const TargetLowering &tli, - unsigned Reg, MVT::ValueType regvt, MVT::ValueType valuevt) - : TLI(&tli), Regs(1, Reg), RegVTs(1, regvt), ValueVTs(1, valuevt) {} RegsForValue(const TargetLowering &tli, const SmallVector ®s, - MVT::ValueType regvt, MVT::ValueType valuevt) - : TLI(&tli), Regs(regs), RegVTs(1, regvt), ValueVTs(1, valuevt) {} + MVT regvt, MVT valuevt) + : TLI(&tli), ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {} RegsForValue(const TargetLowering &tli, const SmallVector ®s, - const SmallVector ®vts, - const SmallVector &valuevts) - : TLI(&tli), Regs(regs), RegVTs(regvts), ValueVTs(valuevts) {} + const SmallVector ®vts, + const SmallVector &valuevts) + : TLI(&tli), ValueVTs(valuevts), RegVTs(regvts), Regs(regs) {} RegsForValue(const TargetLowering &tli, unsigned Reg, const Type *Ty) : TLI(&tli) { ComputeValueVTs(tli, Ty, ValueVTs); - for (unsigned Value = 0; Value != ValueVTs.size(); ++Value) { - MVT::ValueType ValueVT = ValueVTs[Value]; + for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { + MVT ValueVT = ValueVTs[Value]; unsigned NumRegs = TLI->getNumRegisters(ValueVT); - MVT::ValueType RegisterVT = TLI->getRegisterType(ValueVT); + MVT RegisterVT = TLI->getRegisterType(ValueVT); for (unsigned i = 0; i != NumRegs; ++i) Regs.push_back(Reg + i); RegVTs.push_back(RegisterVT); @@ -169,25 +248,34 @@ namespace { } } + /// append - Add the specified values to this one. + void append(const RegsForValue &RHS) { + TLI = RHS.TLI; + ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end()); + RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end()); + Regs.append(RHS.Regs.begin(), RHS.Regs.end()); + } + + /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from /// this value and returns the result as a ValueVTs value. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. - SDOperand getCopyFromRegs(SelectionDAG &DAG, - SDOperand &Chain, SDOperand *Flag) const; + SDValue getCopyFromRegs(SelectionDAG &DAG, + SDValue &Chain, SDValue *Flag) const; /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the /// specified value into the registers specified by this object. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. - void getCopyToRegs(SDOperand Val, SelectionDAG &DAG, - SDOperand &Chain, SDOperand *Flag) const; + void getCopyToRegs(SDValue Val, SelectionDAG &DAG, + SDValue &Chain, SDValue *Flag) const; /// AddInlineAsmOperands - Add this value to the specified inlineasm node /// operand list. This adds the code marker and includes the number of /// values added into it. void AddInlineAsmOperands(unsigned Code, SelectionDAG &DAG, - std::vector &Ops) const; + std::vector &Ops) const; }; } @@ -197,15 +285,16 @@ namespace llvm { /// for the target. ScheduleDAG* createDefaultScheduler(SelectionDAGISel *IS, SelectionDAG *DAG, - MachineBasicBlock *BB) { + MachineBasicBlock *BB, + bool Fast) { TargetLowering &TLI = IS->getTargetLowering(); if (TLI.getSchedulingPreference() == TargetLowering::SchedulingForLatency) { - return createTDListDAGScheduler(IS, DAG, BB); + return createTDListDAGScheduler(IS, DAG, BB, Fast); } else { assert(TLI.getSchedulingPreference() == TargetLowering::SchedulingForRegPressure && "Unknown sched type!"); - return createBURRListDAGScheduler(IS, DAG, BB); + return createBURRListDAGScheduler(IS, DAG, BB, Fast); } } @@ -240,7 +329,7 @@ namespace llvm { SmallSet CatchInfoFound; #endif - unsigned MakeReg(MVT::ValueType VT) { + unsigned MakeReg(MVT VT) { return RegInfo.createVirtualRegister(TLI.getRegClassFor(VT)); } @@ -257,6 +346,16 @@ namespace llvm { assert(R == 0 && "Already initialized this value register!"); return R = CreateRegForValue(V); } + + struct LiveOutInfo { + unsigned NumSignBits; + APInt KnownOne, KnownZero; + LiveOutInfo() : NumSignBits(0) {} + }; + + /// LiveOutRegInfo - Information about live out vregs, indexed by their + /// register number offset by 'FirstVirtualRegister'. + std::vector LiveOutRegInfo; }; } @@ -335,9 +434,9 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, // also creates the initial PHI MachineInstrs, though none of the input // operands are populated. for (BB = Fn.begin(), EB = Fn.end(); BB != EB; ++BB) { - MachineBasicBlock *MBB = new MachineBasicBlock(BB); + MachineBasicBlock *MBB = mf.CreateMachineBasicBlock(BB); MBBMap[BB] = MBB; - MF.getBasicBlockList().push_back(MBB); + MF.push_back(MBB); // Create Machine PHI nodes for LLVM PHI nodes, lowering them as // appropriate. @@ -345,13 +444,19 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, for (BasicBlock::iterator I = BB->begin();(PN = dyn_cast(I)); ++I){ if (PN->use_empty()) continue; - MVT::ValueType VT = TLI.getValueType(PN->getType()); - unsigned NumRegisters = TLI.getNumRegisters(VT); unsigned PHIReg = ValueMap[PN]; assert(PHIReg && "PHI node does not have an assigned virtual register!"); - const TargetInstrInfo *TII = TLI.getTargetMachine().getInstrInfo(); - for (unsigned i = 0; i != NumRegisters; ++i) - BuildMI(MBB, TII->get(TargetInstrInfo::PHI), PHIReg+i); + + SmallVector ValueVTs; + ComputeValueVTs(TLI, PN->getType(), ValueVTs); + for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) { + MVT VT = ValueVTs[vti]; + unsigned NumRegisters = TLI.getNumRegisters(VT); + const TargetInstrInfo *TII = TLI.getTargetMachine().getInstrInfo(); + for (unsigned i = 0; i != NumRegisters; ++i) + BuildMI(MBB, TII->get(TargetInstrInfo::PHI), PHIReg+i); + PHIReg += NumRegisters; + } } } } @@ -359,14 +464,18 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, /// CreateRegForValue - Allocate the appropriate number of virtual registers of /// the correctly promoted or expanded types. Assign these registers /// consecutive vreg numbers and return the first assigned number. +/// +/// In the case that the given value has struct or array type, this function +/// will assign registers for each member or element. +/// unsigned FunctionLoweringInfo::CreateRegForValue(const Value *V) { - SmallVector ValueVTs; + SmallVector ValueVTs; ComputeValueVTs(TLI, V->getType(), ValueVTs); unsigned FirstReg = 0; - for (unsigned Value = 0; Value != ValueVTs.size(); ++Value) { - MVT::ValueType ValueVT = ValueVTs[Value]; - MVT::ValueType RegisterVT = TLI.getRegisterType(ValueVT); + for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { + MVT ValueVT = ValueVTs[Value]; + MVT RegisterVT = TLI.getRegisterType(ValueVT); unsigned NumRegs = TLI.getNumRegisters(ValueVT); for (unsigned i = 0; i != NumRegs; ++i) { @@ -386,20 +495,20 @@ namespace llvm { class SelectionDAGLowering { MachineBasicBlock *CurMBB; - DenseMap NodeMap; + DenseMap NodeMap; /// PendingLoads - Loads are not emitted to the program immediately. We bunch /// them up and then emit token factor nodes when possible. This allows us to /// get simple disambiguation between loads without worrying about alias /// analysis. - std::vector PendingLoads; + SmallVector PendingLoads; /// PendingExports - CopyToReg nodes that copy values to virtual registers /// for export to other blocks need to be emitted before any terminator /// instruction, but they have no other ordering requirements. We bunch them /// up and the emit a single tokenfactor for them just before terminator /// instructions. - std::vector PendingExports; + std::vector PendingExports; /// Case - A struct to record the Value for a switch case, and the /// case's target basic block. @@ -507,19 +616,19 @@ public: /// a store or any other node that may need to be ordered after any /// prior load instructions. /// - SDOperand getRoot() { + SDValue getRoot() { if (PendingLoads.empty()) return DAG.getRoot(); if (PendingLoads.size() == 1) { - SDOperand Root = PendingLoads[0]; + SDValue Root = PendingLoads[0]; DAG.setRoot(Root); PendingLoads.clear(); return Root; } // Otherwise, we have to make a token factor node. - SDOperand Root = DAG.getNode(ISD::TokenFactor, MVT::Other, + SDValue Root = DAG.getNode(ISD::TokenFactor, MVT::Other, &PendingLoads[0], PendingLoads.size()); PendingLoads.clear(); DAG.setRoot(Root); @@ -530,8 +639,8 @@ public: /// PendingLoad items, flush all the PendingExports items. It is necessary /// to do this before emitting a terminator instruction. /// - SDOperand getControlRoot() { - SDOperand Root = DAG.getRoot(); + SDValue getControlRoot() { + SDValue Root = DAG.getRoot(); if (PendingExports.empty()) return Root; @@ -576,14 +685,10 @@ public: void setCurrentBasicBlock(MachineBasicBlock *MBB) { CurMBB = MBB; } - SDOperand getLoadFrom(const Type *Ty, SDOperand Ptr, - const Value *SV, SDOperand Root, - bool isVolatile, unsigned Alignment); - - SDOperand getValue(const Value *V); + SDValue getValue(const Value *V); - void setValue(const Value *V, SDOperand NewN) { - SDOperand &N = NodeMap[V]; + void setValue(const Value *V, SDValue NewN) { + SDValue &N = NodeMap[V]; assert(N.Val == 0 && "Already set a value for this node!"); N = NewN; } @@ -597,7 +702,7 @@ public: unsigned Opc); bool isExportableFromCurrentBlock(Value *V, const BasicBlock *FromBB); void ExportFromCurrentBlock(Value *V); - void LowerCallTo(CallSite CS, SDOperand Callee, bool IsTailCall, + void LowerCallTo(CallSite CS, SDValue Callee, bool IsTailCall, MachineBasicBlock *LandingPad = NULL); // Terminator instructions. @@ -665,6 +770,8 @@ public: void visitAShr(User &I) { visitShift(I, ISD::SRA); } void visitICmp(User &I); void visitFCmp(User &I); + void visitVICmp(User &I); + void visitVFCmp(User &I); // Visit the conversion instructions void visitTrunc(User &I); void visitZExt(User &I); @@ -683,6 +790,9 @@ public: void visitInsertElement(User &I); void visitShuffleVector(User &I); + void visitExtractValue(ExtractValueInst &I); + void visitInsertValue(InsertValueInst &I); + void visitGetElementPtr(User &I); void visitSelect(User &I); @@ -702,8 +812,6 @@ public: void visitVAEnd(CallInst &I); void visitVACopy(CallInst &I); - void visitGetResult(GetResultInst &I); - void visitUserOp1(Instruction &I) { assert(0 && "UserOp1 should not exist at instruction selection time!"); abort(); @@ -712,6 +820,10 @@ public: assert(0 && "UserOp2 should not exist at instruction selection time!"); abort(); } + +private: + inline const char *implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op); + }; } // end namespace llvm @@ -721,32 +833,32 @@ public: /// larger then ValueVT then AssertOp can be used to specify whether the extra /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT /// (ISD::AssertSext). -static SDOperand getCopyFromParts(SelectionDAG &DAG, - const SDOperand *Parts, +static SDValue getCopyFromParts(SelectionDAG &DAG, + const SDValue *Parts, unsigned NumParts, - MVT::ValueType PartVT, - MVT::ValueType ValueVT, + MVT PartVT, + MVT ValueVT, ISD::NodeType AssertOp = ISD::DELETED_NODE) { assert(NumParts > 0 && "No parts to assemble!"); TargetLowering &TLI = DAG.getTargetLoweringInfo(); - SDOperand Val = Parts[0]; + SDValue Val = Parts[0]; if (NumParts > 1) { // Assemble the value from multiple parts. - if (!MVT::isVector(ValueVT)) { - unsigned PartBits = MVT::getSizeInBits(PartVT); - unsigned ValueBits = MVT::getSizeInBits(ValueVT); + if (!ValueVT.isVector()) { + unsigned PartBits = PartVT.getSizeInBits(); + unsigned ValueBits = ValueVT.getSizeInBits(); // Assemble the power of 2 part. unsigned RoundParts = NumParts & (NumParts - 1) ? 1 << Log2_32(NumParts) : NumParts; unsigned RoundBits = PartBits * RoundParts; - MVT::ValueType RoundVT = RoundBits == ValueBits ? - ValueVT : MVT::getIntegerType(RoundBits); - SDOperand Lo, Hi; + MVT RoundVT = RoundBits == ValueBits ? + ValueVT : MVT::getIntegerVT(RoundBits); + SDValue Lo, Hi; if (RoundParts > 2) { - MVT::ValueType HalfVT = MVT::getIntegerType(RoundBits/2); + MVT HalfVT = MVT::getIntegerVT(RoundBits/2); Lo = getCopyFromParts(DAG, Parts, RoundParts/2, PartVT, HalfVT); Hi = getCopyFromParts(DAG, Parts+RoundParts/2, RoundParts/2, PartVT, HalfVT); @@ -761,36 +873,36 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, if (RoundParts < NumParts) { // Assemble the trailing non-power-of-2 part. unsigned OddParts = NumParts - RoundParts; - MVT::ValueType OddVT = MVT::getIntegerType(OddParts * PartBits); + MVT OddVT = MVT::getIntegerVT(OddParts * PartBits); Hi = getCopyFromParts(DAG, Parts+RoundParts, OddParts, PartVT, OddVT); // Combine the round and odd parts. Lo = Val; if (TLI.isBigEndian()) std::swap(Lo, Hi); - MVT::ValueType TotalVT = MVT::getIntegerType(NumParts * PartBits); + MVT TotalVT = MVT::getIntegerVT(NumParts * PartBits); Hi = DAG.getNode(ISD::ANY_EXTEND, TotalVT, Hi); Hi = DAG.getNode(ISD::SHL, TotalVT, Hi, - DAG.getConstant(MVT::getSizeInBits(Lo.getValueType()), + DAG.getConstant(Lo.getValueType().getSizeInBits(), TLI.getShiftAmountTy())); Lo = DAG.getNode(ISD::ZERO_EXTEND, TotalVT, Lo); Val = DAG.getNode(ISD::OR, TotalVT, Lo, Hi); } } else { // Handle a multi-element vector. - MVT::ValueType IntermediateVT, RegisterVT; + MVT IntermediateVT, RegisterVT; unsigned NumIntermediates; unsigned NumRegs = TLI.getVectorTypeBreakdown(ValueVT, IntermediateVT, NumIntermediates, RegisterVT); - assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); + NumParts = NumRegs; // Silence a compiler warning. assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); assert(RegisterVT == Parts[0].getValueType() && "Part type doesn't match part!"); // Assemble the parts into intermediate operands. - SmallVector Ops(NumIntermediates); + SmallVector Ops(NumIntermediates); if (NumIntermediates == NumParts) { // If the register was not expanded, truncate or copy the value, // as appropriate. @@ -810,7 +922,7 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the intermediate // operands. - Val = DAG.getNode(MVT::isVector(IntermediateVT) ? + Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR, ValueVT, &Ops[0], NumIntermediates); } @@ -822,21 +934,21 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, if (PartVT == ValueVT) return Val; - if (MVT::isVector(PartVT)) { - assert(MVT::isVector(ValueVT) && "Unknown vector conversion!"); + if (PartVT.isVector()) { + assert(ValueVT.isVector() && "Unknown vector conversion!"); return DAG.getNode(ISD::BIT_CONVERT, ValueVT, Val); } - if (MVT::isVector(ValueVT)) { - assert(MVT::getVectorElementType(ValueVT) == PartVT && - MVT::getVectorNumElements(ValueVT) == 1 && + if (ValueVT.isVector()) { + assert(ValueVT.getVectorElementType() == PartVT && + ValueVT.getVectorNumElements() == 1 && "Only trivial scalar-to-vector conversions should get here!"); return DAG.getNode(ISD::BUILD_VECTOR, ValueVT, Val); } - if (MVT::isInteger(PartVT) && - MVT::isInteger(ValueVT)) { - if (MVT::getSizeInBits(ValueVT) < MVT::getSizeInBits(PartVT)) { + if (PartVT.isInteger() && + ValueVT.isInteger()) { + if (ValueVT.bitsLT(PartVT)) { // For a truncate, see if we have any information to // indicate whether the truncated bits will always be // zero or sign-extension. @@ -849,65 +961,65 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, } } - if (MVT::isFloatingPoint(PartVT) && MVT::isFloatingPoint(ValueVT)) { - if (ValueVT < Val.getValueType()) + if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { + if (ValueVT.bitsLT(Val.getValueType())) // FP_ROUND's are always exact here. return DAG.getNode(ISD::FP_ROUND, ValueVT, Val, DAG.getIntPtrConstant(1)); return DAG.getNode(ISD::FP_EXTEND, ValueVT, Val); } - if (MVT::getSizeInBits(PartVT) == MVT::getSizeInBits(ValueVT)) + if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) return DAG.getNode(ISD::BIT_CONVERT, ValueVT, Val); assert(0 && "Unknown mismatch!"); - return SDOperand(); + return SDValue(); } /// getCopyToParts - Create a series of nodes that contain the specified value /// split into legal parts. If the parts contain more bits than Val, then, for /// integers, ExtendKind can be used to specify how to generate the extra bits. static void getCopyToParts(SelectionDAG &DAG, - SDOperand Val, - SDOperand *Parts, + SDValue Val, + SDValue *Parts, unsigned NumParts, - MVT::ValueType PartVT, + MVT PartVT, ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { TargetLowering &TLI = DAG.getTargetLoweringInfo(); - MVT::ValueType PtrVT = TLI.getPointerTy(); - MVT::ValueType ValueVT = Val.getValueType(); - unsigned PartBits = MVT::getSizeInBits(PartVT); + MVT PtrVT = TLI.getPointerTy(); + MVT ValueVT = Val.getValueType(); + unsigned PartBits = PartVT.getSizeInBits(); assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!"); if (!NumParts) return; - if (!MVT::isVector(ValueVT)) { + if (!ValueVT.isVector()) { if (PartVT == ValueVT) { assert(NumParts == 1 && "No-op copy with multiple parts!"); Parts[0] = Val; return; } - if (NumParts * PartBits > MVT::getSizeInBits(ValueVT)) { + if (NumParts * PartBits > ValueVT.getSizeInBits()) { // If the parts cover more bits than the value has, promote the value. - if (MVT::isFloatingPoint(PartVT) && MVT::isFloatingPoint(ValueVT)) { + if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { assert(NumParts == 1 && "Do not know what to promote to!"); Val = DAG.getNode(ISD::FP_EXTEND, PartVT, Val); - } else if (MVT::isInteger(PartVT) && MVT::isInteger(ValueVT)) { - ValueVT = MVT::getIntegerType(NumParts * PartBits); + } else if (PartVT.isInteger() && ValueVT.isInteger()) { + ValueVT = MVT::getIntegerVT(NumParts * PartBits); Val = DAG.getNode(ExtendKind, ValueVT, Val); } else { assert(0 && "Unknown mismatch!"); } - } else if (PartBits == MVT::getSizeInBits(ValueVT)) { + } else if (PartBits == ValueVT.getSizeInBits()) { // Different types of the same size. assert(NumParts == 1 && PartVT != ValueVT); Val = DAG.getNode(ISD::BIT_CONVERT, PartVT, Val); - } else if (NumParts * PartBits < MVT::getSizeInBits(ValueVT)) { + } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { // If the parts cover less bits than value has, truncate the value. - if (MVT::isInteger(PartVT) && MVT::isInteger(ValueVT)) { - ValueVT = MVT::getIntegerType(NumParts * PartBits); + if (PartVT.isInteger() && ValueVT.isInteger()) { + ValueVT = MVT::getIntegerVT(NumParts * PartBits); Val = DAG.getNode(ISD::TRUNCATE, ValueVT, Val); } else { assert(0 && "Unknown mismatch!"); @@ -916,7 +1028,7 @@ static void getCopyToParts(SelectionDAG &DAG, // The value may have changed - recompute ValueVT. ValueVT = Val.getValueType(); - assert(NumParts * PartBits == MVT::getSizeInBits(ValueVT) && + assert(NumParts * PartBits == ValueVT.getSizeInBits() && "Failed to tile the value with PartVT!"); if (NumParts == 1) { @@ -928,12 +1040,12 @@ static void getCopyToParts(SelectionDAG &DAG, // Expand the value into multiple parts. if (NumParts & (NumParts - 1)) { // The number of parts is not a power of 2. Split off and copy the tail. - assert(MVT::isInteger(PartVT) && MVT::isInteger(ValueVT) && + assert(PartVT.isInteger() && ValueVT.isInteger() && "Do not know what to expand to!"); unsigned RoundParts = 1 << Log2_32(NumParts); unsigned RoundBits = RoundParts * PartBits; unsigned OddParts = NumParts - RoundParts; - SDOperand OddVal = DAG.getNode(ISD::SRL, ValueVT, Val, + SDValue OddVal = DAG.getNode(ISD::SRL, ValueVT, Val, DAG.getConstant(RoundBits, TLI.getShiftAmountTy())); getCopyToParts(DAG, OddVal, Parts + RoundParts, OddParts, PartVT); @@ -941,21 +1053,21 @@ static void getCopyToParts(SelectionDAG &DAG, // The odd parts were reversed by getCopyToParts - unreverse them. std::reverse(Parts + RoundParts, Parts + NumParts); NumParts = RoundParts; - ValueVT = MVT::getIntegerType(NumParts * PartBits); + ValueVT = MVT::getIntegerVT(NumParts * PartBits); Val = DAG.getNode(ISD::TRUNCATE, ValueVT, Val); } // The number of parts is a power of 2. Repeatedly bisect the value using // EXTRACT_ELEMENT. Parts[0] = DAG.getNode(ISD::BIT_CONVERT, - MVT::getIntegerType(MVT::getSizeInBits(ValueVT)), + MVT::getIntegerVT(ValueVT.getSizeInBits()), Val); for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { for (unsigned i = 0; i < NumParts; i += StepSize) { unsigned ThisBits = StepSize * PartBits / 2; - MVT::ValueType ThisVT = MVT::getIntegerType (ThisBits); - SDOperand &Part0 = Parts[i]; - SDOperand &Part1 = Parts[i+StepSize/2]; + MVT ThisVT = MVT::getIntegerVT (ThisBits); + SDValue &Part0 = Parts[i]; + SDValue &Part1 = Parts[i+StepSize/2]; Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, ThisVT, Part0, DAG.getConstant(1, PtrVT)); @@ -978,11 +1090,11 @@ static void getCopyToParts(SelectionDAG &DAG, // Vector ValueVT. if (NumParts == 1) { if (PartVT != ValueVT) { - if (MVT::isVector(PartVT)) { + if (PartVT.isVector()) { Val = DAG.getNode(ISD::BIT_CONVERT, PartVT, Val); } else { - assert(MVT::getVectorElementType(ValueVT) == PartVT && - MVT::getVectorNumElements(ValueVT) == 1 && + assert(ValueVT.getVectorElementType() == PartVT && + ValueVT.getVectorNumElements() == 1 && "Only trivial vector-to-scalar conversions should get here!"); Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, PartVT, Val, DAG.getConstant(0, PtrVT)); @@ -994,21 +1106,22 @@ static void getCopyToParts(SelectionDAG &DAG, } // Handle a multi-element vector. - MVT::ValueType IntermediateVT, RegisterVT; + MVT IntermediateVT, RegisterVT; unsigned NumIntermediates; unsigned NumRegs = DAG.getTargetLoweringInfo() .getVectorTypeBreakdown(ValueVT, IntermediateVT, NumIntermediates, RegisterVT); - unsigned NumElements = MVT::getVectorNumElements(ValueVT); + unsigned NumElements = ValueVT.getVectorNumElements(); assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); + NumParts = NumRegs; // Silence a compiler warning. assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); // Split the vector into intermediate operands. - SmallVector Ops(NumIntermediates); + SmallVector Ops(NumIntermediates); for (unsigned i = 0; i != NumIntermediates; ++i) - if (MVT::isVector(IntermediateVT)) + if (IntermediateVT.isVector()) Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, IntermediateVT, Val, DAG.getConstant(i * (NumElements / NumIntermediates), @@ -1036,13 +1149,12 @@ static void getCopyToParts(SelectionDAG &DAG, } -SDOperand SelectionDAGLowering::getValue(const Value *V) { - SDOperand &N = NodeMap[V]; +SDValue SelectionDAGLowering::getValue(const Value *V) { + SDValue &N = NodeMap[V]; if (N.Val) return N; - const Type *VTy = V->getType(); if (Constant *C = const_cast(dyn_cast(V))) { - MVT::ValueType VT = TLI.getValueType(VTy, true); + MVT VT = TLI.getValueType(V->getType(), true); if (ConstantInt *CI = dyn_cast(C)) return N = DAG.getConstant(CI->getValue(), VT); @@ -1056,48 +1168,71 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { if (ConstantFP *CFP = dyn_cast(C)) return N = DAG.getConstantFP(CFP->getValueAPF(), VT); - if (isa(C)) { - if (!isa(VTy)) - return N = DAG.getNode(ISD::UNDEF, VT); - - // Create a BUILD_VECTOR of undef nodes. - const VectorType *PTy = cast(VTy); - unsigned NumElements = PTy->getNumElements(); - MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); - - SmallVector Ops; - Ops.assign(NumElements, DAG.getNode(ISD::UNDEF, PVT)); - - // Create a VConstant node with generic Vector type. - MVT::ValueType VT = MVT::getVectorType(PVT, NumElements); - return N = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); - } + if (isa(C) && !isa(V->getType()) && + !V->getType()->isAggregateType()) + return N = DAG.getNode(ISD::UNDEF, VT); if (ConstantExpr *CE = dyn_cast(C)) { visit(CE->getOpcode(), *CE); - SDOperand N1 = NodeMap[V]; + SDValue N1 = NodeMap[V]; assert(N1.Val && "visit didn't populate the ValueMap!"); return N1; } - const VectorType *VecTy = cast(VTy); + if (isa(C) || isa(C)) { + SmallVector Constants; + for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end(); + OI != OE; ++OI) { + SDNode *Val = getValue(*OI).Val; + for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) + Constants.push_back(SDValue(Val, i)); + } + return DAG.getMergeValues(&Constants[0], Constants.size()); + } + + if (isa(C->getType()) || isa(C->getType())) { + assert((isa(C) || isa(C)) && + "Unknown struct or array constant!"); + + SmallVector ValueVTs; + ComputeValueVTs(TLI, C->getType(), ValueVTs); + unsigned NumElts = ValueVTs.size(); + if (NumElts == 0) + return SDValue(); // empty struct + SmallVector Constants(NumElts); + for (unsigned i = 0; i != NumElts; ++i) { + MVT EltVT = ValueVTs[i]; + if (isa(C)) + Constants[i] = DAG.getNode(ISD::UNDEF, EltVT); + else if (EltVT.isFloatingPoint()) + Constants[i] = DAG.getConstantFP(0, EltVT); + else + Constants[i] = DAG.getConstant(0, EltVT); + } + return DAG.getMergeValues(&Constants[0], NumElts); + } + + const VectorType *VecTy = cast(V->getType()); unsigned NumElements = VecTy->getNumElements(); - MVT::ValueType PVT = TLI.getValueType(VecTy->getElementType()); - // Now that we know the number and type of the elements, push a - // Constant or ConstantFP node onto the ops list for each element of - // the vector constant. - SmallVector Ops; + // Now that we know the number and type of the elements, get that number of + // elements into the Ops array based on what kind of constant it is. + SmallVector Ops; if (ConstantVector *CP = dyn_cast(C)) { for (unsigned i = 0; i != NumElements; ++i) Ops.push_back(getValue(CP->getOperand(i))); } else { - assert(isa(C) && "Unknown vector constant!"); - SDOperand Op; - if (MVT::isFloatingPoint(PVT)) - Op = DAG.getConstantFP(0, PVT); + assert((isa(C) || isa(C)) && + "Unknown vector constant!"); + MVT EltVT = TLI.getValueType(VecTy->getElementType()); + + SDValue Op; + if (isa(C)) + Op = DAG.getNode(ISD::UNDEF, EltVT); + else if (EltVT.isFloatingPoint()) + Op = DAG.getConstantFP(0, EltVT); else - Op = DAG.getConstant(0, PVT); + Op = DAG.getConstant(0, EltVT); Ops.assign(NumElements, Op); } @@ -1117,8 +1252,8 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { unsigned InReg = FuncInfo.ValueMap[V]; assert(InReg && "Value not in map!"); - RegsForValue RFV(TLI, InReg, VTy); - SDOperand Chain = DAG.getEntryNode(); + RegsForValue RFV(TLI, InReg, V->getType()); + SDValue Chain = DAG.getEntryNode(); return RFV.getCopyFromRegs(DAG, Chain, NULL); } @@ -1129,36 +1264,42 @@ void SelectionDAGLowering::visitRet(ReturnInst &I) { return; } - SmallVector NewValues; + SmallVector NewValues; NewValues.push_back(getControlRoot()); for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { - SDOperand RetOp = getValue(I.getOperand(i)); - MVT::ValueType VT = RetOp.getValueType(); - - // FIXME: C calling convention requires the return type to be promoted to - // at least 32-bit. But this is not necessary for non-C calling conventions. - if (MVT::isInteger(VT)) { - MVT::ValueType MinVT = TLI.getRegisterType(MVT::i32); - if (MVT::getSizeInBits(VT) < MVT::getSizeInBits(MinVT)) - VT = MinVT; - } - - unsigned NumParts = TLI.getNumRegisters(VT); - MVT::ValueType PartVT = TLI.getRegisterType(VT); - SmallVector Parts(NumParts); - ISD::NodeType ExtendKind = ISD::ANY_EXTEND; - - const Function *F = I.getParent()->getParent(); - if (F->paramHasAttr(0, ParamAttr::SExt)) - ExtendKind = ISD::SIGN_EXTEND; - else if (F->paramHasAttr(0, ParamAttr::ZExt)) - ExtendKind = ISD::ZERO_EXTEND; - - getCopyToParts(DAG, RetOp, &Parts[0], NumParts, PartVT, ExtendKind); + SDValue RetOp = getValue(I.getOperand(i)); + + SmallVector ValueVTs; + ComputeValueVTs(TLI, I.getOperand(i)->getType(), ValueVTs); + for (unsigned j = 0, f = ValueVTs.size(); j != f; ++j) { + MVT VT = ValueVTs[j]; + + // FIXME: C calling convention requires the return type to be promoted to + // at least 32-bit. But this is not necessary for non-C calling conventions. + if (VT.isInteger()) { + MVT MinVT = TLI.getRegisterType(MVT::i32); + if (VT.bitsLT(MinVT)) + VT = MinVT; + } - for (unsigned i = 0; i < NumParts; ++i) { - NewValues.push_back(Parts[i]); - NewValues.push_back(DAG.getArgFlags(ISD::ArgFlagsTy())); + unsigned NumParts = TLI.getNumRegisters(VT); + MVT PartVT = TLI.getRegisterType(VT); + SmallVector Parts(NumParts); + ISD::NodeType ExtendKind = ISD::ANY_EXTEND; + + const Function *F = I.getParent()->getParent(); + if (F->paramHasAttr(0, ParamAttr::SExt)) + ExtendKind = ISD::SIGN_EXTEND; + else if (F->paramHasAttr(0, ParamAttr::ZExt)) + ExtendKind = ISD::ZERO_EXTEND; + + getCopyToParts(DAG, SDValue(RetOp.Val, RetOp.ResNo + j), + &Parts[0], NumParts, PartVT, ExtendKind); + + for (unsigned i = 0; i < NumParts; ++i) { + NewValues.push_back(Parts[i]); + NewValues.push_back(DAG.getArgFlags(ISD::ArgFlagsTy())); + } } } DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, @@ -1264,8 +1405,8 @@ void SelectionDAGLowering::FindMergedConditions(Value *Cond, case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break; case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break; case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break; - case FCmpInst::FCMP_ORD: FOC = ISD::SETEQ; FPC = ISD::SETO; break; - case FCmpInst::FCMP_UNO: FOC = ISD::SETNE; FPC = ISD::SETUO; break; + case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break; + case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break; case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break; case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break; case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break; @@ -1299,8 +1440,9 @@ void SelectionDAGLowering::FindMergedConditions(Value *Cond, // Create TmpBB after CurBB. MachineFunction::iterator BBI = CurBB; - MachineBasicBlock *TmpBB = new MachineBasicBlock(CurBB->getBasicBlock()); - CurBB->getParent()->getBasicBlockList().insert(++BBI, TmpBB); + MachineFunction &MF = DAG.getMachineFunction(); + MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock()); + CurBB->getParent()->insert(++BBI, TmpBB); if (Opc == Instruction::Or) { // Codegen X | Y as: @@ -1365,13 +1507,13 @@ void SelectionDAGLowering::visitBr(BranchInst &I) { NextBlock = BBI; if (I.isUnconditional()) { + // Update machine-CFG edges. + CurMBB->addSuccessor(Succ0MBB); + // If this is not a fall-through branch, emit the branch. if (Succ0MBB != NextBlock) DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getControlRoot(), DAG.getBasicBlock(Succ0MBB))); - - // Update machine-CFG edges. - CurMBB->addSuccessor(Succ0MBB); return; } @@ -1421,7 +1563,7 @@ void SelectionDAGLowering::visitBr(BranchInst &I) { // Okay, we decided not to do this, remove any inserted MBB's and clear // SwitchCases. for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) - CurMBB->getParent()->getBasicBlockList().erase(SwitchCases[i].ThisBB); + CurMBB->getParent()->erase(SwitchCases[i].ThisBB); SwitchCases.clear(); } @@ -1438,8 +1580,8 @@ void SelectionDAGLowering::visitBr(BranchInst &I) { /// visitSwitchCase - Emits the necessary code to represent a single node in /// the binary search tree resulting from lowering a switch instruction. void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) { - SDOperand Cond; - SDOperand CondLHS = getValue(CB.CmpLHS); + SDValue Cond; + SDValue CondLHS = getValue(CB.CmpLHS); // Build the setcc now. if (CB.CmpMHS == NULL) { @@ -1448,7 +1590,7 @@ void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) { if (CB.CmpRHS == ConstantInt::getTrue() && CB.CC == ISD::SETEQ) Cond = CondLHS; else if (CB.CmpRHS == ConstantInt::getFalse() && CB.CC == ISD::SETEQ) { - SDOperand True = DAG.getConstant(1, CondLHS.getValueType()); + SDValue True = DAG.getConstant(1, CondLHS.getValueType()); Cond = DAG.getNode(ISD::XOR, CondLHS.getValueType(), CondLHS, True); } else Cond = DAG.getSetCC(MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC); @@ -1458,19 +1600,22 @@ void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) { uint64_t Low = cast(CB.CmpLHS)->getSExtValue(); uint64_t High = cast(CB.CmpRHS)->getSExtValue(); - SDOperand CmpOp = getValue(CB.CmpMHS); - MVT::ValueType VT = CmpOp.getValueType(); + SDValue CmpOp = getValue(CB.CmpMHS); + MVT VT = CmpOp.getValueType(); if (cast(CB.CmpLHS)->isMinValue(true)) { Cond = DAG.getSetCC(MVT::i1, CmpOp, DAG.getConstant(High, VT), ISD::SETLE); } else { - SDOperand SUB = DAG.getNode(ISD::SUB, VT, CmpOp, DAG.getConstant(Low, VT)); + SDValue SUB = DAG.getNode(ISD::SUB, VT, CmpOp, DAG.getConstant(Low, VT)); Cond = DAG.getSetCC(MVT::i1, SUB, DAG.getConstant(High-Low, VT), ISD::SETULE); } - } + // Update successor info + CurMBB->addSuccessor(CB.TrueBB); + CurMBB->addSuccessor(CB.FalseBB); + // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; @@ -1482,28 +1627,38 @@ void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) { // fall through to the lhs instead of the rhs block. if (CB.TrueBB == NextBlock) { std::swap(CB.TrueBB, CB.FalseBB); - SDOperand True = DAG.getConstant(1, Cond.getValueType()); + SDValue True = DAG.getConstant(1, Cond.getValueType()); Cond = DAG.getNode(ISD::XOR, Cond.getValueType(), Cond, True); } - SDOperand BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, getControlRoot(), Cond, + SDValue BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, getControlRoot(), Cond, DAG.getBasicBlock(CB.TrueBB)); - if (CB.FalseBB == NextBlock) + + // If the branch was constant folded, fix up the CFG. + if (BrCond.getOpcode() == ISD::BR) { + if (!DisableCorrectBranchFolding) + CurMBB->removeSuccessor(CB.FalseBB); DAG.setRoot(BrCond); - else - DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, BrCond, - DAG.getBasicBlock(CB.FalseBB))); - // Update successor info - CurMBB->addSuccessor(CB.TrueBB); - CurMBB->addSuccessor(CB.FalseBB); + } else { + // Otherwise, go ahead and insert the false branch. + if (BrCond == getControlRoot()) + if (!DisableCorrectBranchFolding) + CurMBB->removeSuccessor(CB.TrueBB); + + if (CB.FalseBB == NextBlock) + DAG.setRoot(BrCond); + else + DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, BrCond, + DAG.getBasicBlock(CB.FalseBB))); + } } /// visitJumpTable - Emit JumpTable node in the current MBB void SelectionDAGLowering::visitJumpTable(SelectionDAGISel::JumpTable &JT) { // Emit the code for the jump table assert(JT.Reg != -1U && "Should lower JT Header first!"); - MVT::ValueType PTy = TLI.getPointerTy(); - SDOperand Index = DAG.getCopyFromReg(getControlRoot(), JT.Reg, PTy); - SDOperand Table = DAG.getJumpTable(JT.JTI, PTy); + MVT PTy = TLI.getPointerTy(); + SDValue Index = DAG.getCopyFromReg(getControlRoot(), JT.Reg, PTy); + SDValue Table = DAG.getJumpTable(JT.JTI, PTy); DAG.setRoot(DAG.getNode(ISD::BR_JT, MVT::Other, Index.getValue(1), Table, Index)); return; @@ -1516,9 +1671,9 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, // Subtract the lowest switch case value from the value being switched on // and conditional branch to default mbb if the result is greater than the // difference between smallest and largest cases. - SDOperand SwitchOp = getValue(JTH.SValue); - MVT::ValueType VT = SwitchOp.getValueType(); - SDOperand SUB = DAG.getNode(ISD::SUB, VT, SwitchOp, + SDValue SwitchOp = getValue(JTH.SValue); + MVT VT = SwitchOp.getValueType(); + SDValue SUB = DAG.getNode(ISD::SUB, VT, SwitchOp, DAG.getConstant(JTH.First, VT)); // The SDNode we just created, which holds the value being switched on @@ -1526,19 +1681,19 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, // register so it can be used as an index into the jump table in a // subsequent basic block. This value may be smaller or larger than the // target's pointer type, and therefore require extension or truncating. - if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy())) + if (VT.bitsGT(TLI.getPointerTy())) SwitchOp = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), SUB); else SwitchOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), SUB); unsigned JumpTableReg = FuncInfo.MakeReg(TLI.getPointerTy()); - SDOperand CopyTo = DAG.getCopyToReg(getControlRoot(), JumpTableReg, SwitchOp); + SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), JumpTableReg, SwitchOp); JT.Reg = JumpTableReg; // Emit the range check for the jump table, and branch to the default // block for the switch statement if the value being switched on exceeds // the largest case in the switch. - SDOperand CMP = DAG.getSetCC(TLI.getSetCCResultType(SUB), SUB, + SDValue CMP = DAG.getSetCC(TLI.getSetCCResultType(SUB), SUB, DAG.getConstant(JTH.Last-JTH.First,VT), ISD::SETUGT); @@ -1549,7 +1704,7 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, if (++BBI != CurMBB->getParent()->end()) NextBlock = BBI; - SDOperand BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, CMP, + SDValue BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, CMP, DAG.getBasicBlock(JT.Default)); if (JT.MBB == NextBlock) @@ -1565,34 +1720,31 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, /// suitable for "bit tests" void SelectionDAGLowering::visitBitTestHeader(SelectionDAGISel::BitTestBlock &B) { // Subtract the minimum value - SDOperand SwitchOp = getValue(B.SValue); - MVT::ValueType VT = SwitchOp.getValueType(); - SDOperand SUB = DAG.getNode(ISD::SUB, VT, SwitchOp, + SDValue SwitchOp = getValue(B.SValue); + MVT VT = SwitchOp.getValueType(); + SDValue SUB = DAG.getNode(ISD::SUB, VT, SwitchOp, DAG.getConstant(B.First, VT)); // Check range - SDOperand RangeCmp = DAG.getSetCC(TLI.getSetCCResultType(SUB), SUB, + SDValue RangeCmp = DAG.getSetCC(TLI.getSetCCResultType(SUB), SUB, DAG.getConstant(B.Range, VT), ISD::SETUGT); - SDOperand ShiftOp; - if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getShiftAmountTy())) + SDValue ShiftOp; + if (VT.bitsGT(TLI.getShiftAmountTy())) ShiftOp = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), SUB); else ShiftOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getShiftAmountTy(), SUB); // Make desired shift - SDOperand SwitchVal = DAG.getNode(ISD::SHL, TLI.getPointerTy(), + SDValue SwitchVal = DAG.getNode(ISD::SHL, TLI.getPointerTy(), DAG.getConstant(1, TLI.getPointerTy()), ShiftOp); unsigned SwitchReg = FuncInfo.MakeReg(TLI.getPointerTy()); - SDOperand CopyTo = DAG.getCopyToReg(getControlRoot(), SwitchReg, SwitchVal); + SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), SwitchReg, SwitchVal); B.Reg = SwitchReg; - SDOperand BrRange = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, RangeCmp, - DAG.getBasicBlock(B.Default)); - // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; @@ -1601,15 +1753,19 @@ void SelectionDAGLowering::visitBitTestHeader(SelectionDAGISel::BitTestBlock &B) NextBlock = BBI; MachineBasicBlock* MBB = B.Cases[0].ThisBB; + + CurMBB->addSuccessor(B.Default); + CurMBB->addSuccessor(MBB); + + SDValue BrRange = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, RangeCmp, + DAG.getBasicBlock(B.Default)); + if (MBB == NextBlock) DAG.setRoot(BrRange); else DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, CopyTo, DAG.getBasicBlock(MBB))); - CurMBB->addSuccessor(B.Default); - CurMBB->addSuccessor(MBB); - return; } @@ -1618,16 +1774,19 @@ void SelectionDAGLowering::visitBitTestCase(MachineBasicBlock* NextMBB, unsigned Reg, SelectionDAGISel::BitTestCase &B) { // Emit bit tests and jumps - SDOperand SwitchVal = DAG.getCopyFromReg(getControlRoot(), Reg, TLI.getPointerTy()); + SDValue SwitchVal = DAG.getCopyFromReg(getControlRoot(), Reg, + TLI.getPointerTy()); - SDOperand AndOp = DAG.getNode(ISD::AND, TLI.getPointerTy(), - SwitchVal, - DAG.getConstant(B.Mask, - TLI.getPointerTy())); - SDOperand AndCmp = DAG.getSetCC(TLI.getSetCCResultType(AndOp), AndOp, + SDValue AndOp = DAG.getNode(ISD::AND, TLI.getPointerTy(), SwitchVal, + DAG.getConstant(B.Mask, TLI.getPointerTy())); + SDValue AndCmp = DAG.getSetCC(TLI.getSetCCResultType(AndOp), AndOp, DAG.getConstant(0, TLI.getPointerTy()), ISD::SETNE); - SDOperand BrAnd = DAG.getNode(ISD::BRCOND, MVT::Other, getControlRoot(), + + CurMBB->addSuccessor(B.TargetBB); + CurMBB->addSuccessor(NextMBB); + + SDValue BrAnd = DAG.getNode(ISD::BRCOND, MVT::Other, getControlRoot(), AndCmp, DAG.getBasicBlock(B.TargetBB)); // Set NextBlock to be the MBB immediately after the current one, if any. @@ -1643,9 +1802,6 @@ void SelectionDAGLowering::visitBitTestCase(MachineBasicBlock* NextMBB, DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, BrAnd, DAG.getBasicBlock(NextMBB))); - CurMBB->addSuccessor(B.TargetBB); - CurMBB->addSuccessor(NextMBB); - return; } @@ -1667,13 +1823,13 @@ void SelectionDAGLowering::visitInvoke(InvokeInst &I) { CopyValueToVirtualRegister(&I, VMI->second); } - // Drop into normal successor. - DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getControlRoot(), - DAG.getBasicBlock(Return))); - // Update successor info CurMBB->addSuccessor(Return); CurMBB->addSuccessor(LandingPad); + + // Drop into normal successor. + DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getControlRoot(), + DAG.getBasicBlock(Return))); } void SelectionDAGLowering::visitUnwind(UnwindInst &I) { @@ -1727,8 +1883,8 @@ bool SelectionDAGLowering::handleSmallSwitchRange(CaseRec& CR, for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) { MachineBasicBlock *FallThrough; if (I != E-1) { - FallThrough = new MachineBasicBlock(CurBlock->getBasicBlock()); - CurMF->getBasicBlockList().insert(BBI, FallThrough); + FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock()); + CurMF->insert(BBI, FallThrough); } else { // If the last case doesn't match, go to the default block. FallThrough = Default; @@ -1763,8 +1919,9 @@ bool SelectionDAGLowering::handleSmallSwitchRange(CaseRec& CR, } static inline bool areJTsAllowed(const TargetLowering &TLI) { - return (TLI.isOperationLegal(ISD::BR_JT, MVT::Other) || - TLI.isOperationLegal(ISD::BRIND, MVT::Other)); + return !DisableJumpTables && + (TLI.isOperationLegal(ISD::BR_JT, MVT::Other) || + TLI.isOperationLegal(ISD::BRIND, MVT::Other)); } /// handleJTSwitchCase - Emit jumptable for current switch case range @@ -1811,8 +1968,8 @@ bool SelectionDAGLowering::handleJTSwitchCase(CaseRec& CR, // of the jump table, and jumping to it. Update successor information; // we will either branch to the default case for the switch, or the jump // table. - MachineBasicBlock *JumpTableBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, JumpTableBB); + MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, JumpTableBB); CR.CaseBB->addSuccessor(Default); CR.CaseBB->addSuccessor(JumpTableBB); @@ -1949,8 +2106,8 @@ bool SelectionDAGLowering::handleBTSplitSwitchCase(CaseRec& CR, (cast(CR.GE)->getSExtValue() + 1LL)) { TrueBB = LHSR.first->BB; } else { - TrueBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, TrueBB); + TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, TrueBB); WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR)); } @@ -1963,8 +2120,8 @@ bool SelectionDAGLowering::handleBTSplitSwitchCase(CaseRec& CR, (cast(CR.LT)->getSExtValue() - 1LL)) { FalseBB = RHSR.first->BB; } else { - FalseBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, FalseBB); + FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, FalseBB); WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR)); } @@ -1989,7 +2146,7 @@ bool SelectionDAGLowering::handleBitTestsSwitchCase(CaseRec& CR, CaseRecVector& WorkList, Value* SV, MachineBasicBlock* Default){ - unsigned IntPtrBits = MVT::getSizeInBits(TLI.getPointerTy()); + unsigned IntPtrBits = TLI.getPointerTy().getSizeInBits(); Case& FrontCase = *CR.Range.first; Case& BackCase = *(CR.Range.second-1); @@ -2086,8 +2243,8 @@ bool SelectionDAGLowering::handleBitTestsSwitchCase(CaseRec& CR, DOUT << "Mask: " << CasesBits[i].Mask << ", Bits: " << CasesBits[i].Bits << ", BB: " << CasesBits[i].BB << "\n"; - MachineBasicBlock *CaseBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, CaseBB); + MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, CaseBB); BTC.push_back(SelectionDAGISel::BitTestCase(CasesBits[i].Mask, CaseBB, CasesBits[i].BB)); @@ -2162,11 +2319,11 @@ void SelectionDAGLowering::visitSwitch(SwitchInst &SI) { // Update machine-CFG edges. // If this is not a fall-through branch, emit the branch. + CurMBB->addSuccessor(Default); if (Default != NextBlock) DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getControlRoot(), DAG.getBasicBlock(Default))); - - CurMBB->addSuccessor(Default); + return; } @@ -2225,7 +2382,7 @@ void SelectionDAGLowering::visitSub(User &I) { std::vector NZ(VL, ConstantFP::getNegativeZero(ElTy)); Constant *CNZ = ConstantVector::get(&NZ[0], NZ.size()); if (CV == CNZ) { - SDOperand Op2 = getValue(I.getOperand(1)); + SDValue Op2 = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2)); return; } @@ -2235,7 +2392,7 @@ void SelectionDAGLowering::visitSub(User &I) { if (Ty->isFloatingPoint()) { if (ConstantFP *CFP = dyn_cast(I.getOperand(0))) if (CFP->isExactlyValue(ConstantFP::getNegativeZero(Ty)->getValueAPF())) { - SDOperand Op2 = getValue(I.getOperand(1)); + SDValue Op2 = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2)); return; } @@ -2245,21 +2402,21 @@ void SelectionDAGLowering::visitSub(User &I) { } void SelectionDAGLowering::visitBinary(User &I, unsigned OpCode) { - SDOperand Op1 = getValue(I.getOperand(0)); - SDOperand Op2 = getValue(I.getOperand(1)); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(OpCode, Op1.getValueType(), Op1, Op2)); } void SelectionDAGLowering::visitShift(User &I, unsigned Opcode) { - SDOperand Op1 = getValue(I.getOperand(0)); - SDOperand Op2 = getValue(I.getOperand(1)); - - if (MVT::getSizeInBits(TLI.getShiftAmountTy()) < - MVT::getSizeInBits(Op2.getValueType())) - Op2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Op2); - else if (TLI.getShiftAmountTy() > Op2.getValueType()) - Op2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Op2); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); + if (!isa(I.getType())) { + if (TLI.getShiftAmountTy().bitsLT(Op2.getValueType())) + Op2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Op2); + else if (TLI.getShiftAmountTy().bitsGT(Op2.getValueType())) + Op2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Op2); + } setValue(&I, DAG.getNode(Opcode, Op1.getValueType(), Op1, Op2)); } @@ -2270,8 +2427,8 @@ void SelectionDAGLowering::visitICmp(User &I) { predicate = IC->getPredicate(); else if (ConstantExpr *IC = dyn_cast(&I)) predicate = ICmpInst::Predicate(IC->getPredicate()); - SDOperand Op1 = getValue(I.getOperand(0)); - SDOperand Op2 = getValue(I.getOperand(1)); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); ISD::CondCode Opcode; switch (predicate) { case ICmpInst::ICMP_EQ : Opcode = ISD::SETEQ; break; @@ -2298,8 +2455,8 @@ void SelectionDAGLowering::visitFCmp(User &I) { predicate = FC->getPredicate(); else if (ConstantExpr *FC = dyn_cast(&I)) predicate = FCmpInst::Predicate(FC->getPredicate()); - SDOperand Op1 = getValue(I.getOperand(0)); - SDOperand Op2 = getValue(I.getOperand(1)); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); ISD::CondCode Condition, FOC, FPC; switch (predicate) { case FCmpInst::FCMP_FALSE: FOC = FPC = ISD::SETFALSE; break; @@ -2309,8 +2466,8 @@ void SelectionDAGLowering::visitFCmp(User &I) { case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break; case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break; case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break; - case FCmpInst::FCMP_ORD: FOC = ISD::SETEQ; FPC = ISD::SETO; break; - case FCmpInst::FCMP_UNO: FOC = ISD::SETNE; FPC = ISD::SETUO; break; + case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break; + case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break; case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break; case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break; case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break; @@ -2330,10 +2487,79 @@ void SelectionDAGLowering::visitFCmp(User &I) { setValue(&I, DAG.getSetCC(MVT::i1, Op1, Op2, Condition)); } +void SelectionDAGLowering::visitVICmp(User &I) { + ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE; + if (VICmpInst *IC = dyn_cast(&I)) + predicate = IC->getPredicate(); + else if (ConstantExpr *IC = dyn_cast(&I)) + predicate = ICmpInst::Predicate(IC->getPredicate()); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); + ISD::CondCode Opcode; + switch (predicate) { + case ICmpInst::ICMP_EQ : Opcode = ISD::SETEQ; break; + case ICmpInst::ICMP_NE : Opcode = ISD::SETNE; break; + case ICmpInst::ICMP_UGT : Opcode = ISD::SETUGT; break; + case ICmpInst::ICMP_UGE : Opcode = ISD::SETUGE; break; + case ICmpInst::ICMP_ULT : Opcode = ISD::SETULT; break; + case ICmpInst::ICMP_ULE : Opcode = ISD::SETULE; break; + case ICmpInst::ICMP_SGT : Opcode = ISD::SETGT; break; + case ICmpInst::ICMP_SGE : Opcode = ISD::SETGE; break; + case ICmpInst::ICMP_SLT : Opcode = ISD::SETLT; break; + case ICmpInst::ICMP_SLE : Opcode = ISD::SETLE; break; + default: + assert(!"Invalid ICmp predicate value"); + Opcode = ISD::SETEQ; + break; + } + setValue(&I, DAG.getVSetCC(Op1.getValueType(), Op1, Op2, Opcode)); +} + +void SelectionDAGLowering::visitVFCmp(User &I) { + FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE; + if (VFCmpInst *FC = dyn_cast(&I)) + predicate = FC->getPredicate(); + else if (ConstantExpr *FC = dyn_cast(&I)) + predicate = FCmpInst::Predicate(FC->getPredicate()); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); + ISD::CondCode Condition, FOC, FPC; + switch (predicate) { + case FCmpInst::FCMP_FALSE: FOC = FPC = ISD::SETFALSE; break; + case FCmpInst::FCMP_OEQ: FOC = ISD::SETEQ; FPC = ISD::SETOEQ; break; + case FCmpInst::FCMP_OGT: FOC = ISD::SETGT; FPC = ISD::SETOGT; break; + case FCmpInst::FCMP_OGE: FOC = ISD::SETGE; FPC = ISD::SETOGE; break; + case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break; + case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break; + case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break; + case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break; + case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break; + case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break; + case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break; + case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break; + case FCmpInst::FCMP_ULT: FOC = ISD::SETLT; FPC = ISD::SETULT; break; + case FCmpInst::FCMP_ULE: FOC = ISD::SETLE; FPC = ISD::SETULE; break; + case FCmpInst::FCMP_UNE: FOC = ISD::SETNE; FPC = ISD::SETUNE; break; + case FCmpInst::FCMP_TRUE: FOC = FPC = ISD::SETTRUE; break; + default: + assert(!"Invalid VFCmp predicate value"); + FOC = FPC = ISD::SETFALSE; + break; + } + if (FiniteOnlyFPMath()) + Condition = FOC; + else + Condition = FPC; + + MVT DestVT = TLI.getValueType(I.getType()); + + setValue(&I, DAG.getVSetCC(DestVT, Op1, Op2, Condition)); +} + void SelectionDAGLowering::visitSelect(User &I) { - SDOperand Cond = getValue(I.getOperand(0)); - SDOperand TrueVal = getValue(I.getOperand(1)); - SDOperand FalseVal = getValue(I.getOperand(2)); + SDValue Cond = getValue(I.getOperand(0)); + SDValue TrueVal = getValue(I.getOperand(1)); + SDValue FalseVal = getValue(I.getOperand(2)); setValue(&I, DAG.getNode(ISD::SELECT, TrueVal.getValueType(), Cond, TrueVal, FalseVal)); } @@ -2341,77 +2567,77 @@ void SelectionDAGLowering::visitSelect(User &I) { void SelectionDAGLowering::visitTrunc(User &I) { // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::TRUNCATE, DestVT, N)); } void SelectionDAGLowering::visitZExt(User &I) { // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). // ZExt also can't be a cast to bool for same reason. So, nothing much to do - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, DestVT, N)); } void SelectionDAGLowering::visitSExt(User &I) { // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). // SExt also can't be a cast to bool for same reason. So, nothing much to do - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, DestVT, N)); } void SelectionDAGLowering::visitFPTrunc(User &I) { // FPTrunc is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_ROUND, DestVT, N, DAG.getIntPtrConstant(0))); } void SelectionDAGLowering::visitFPExt(User &I){ // FPTrunc is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_EXTEND, DestVT, N)); } void SelectionDAGLowering::visitFPToUI(User &I) { // FPToUI is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_TO_UINT, DestVT, N)); } void SelectionDAGLowering::visitFPToSI(User &I) { // FPToSI is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_TO_SINT, DestVT, N)); } void SelectionDAGLowering::visitUIToFP(User &I) { // UIToFP is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::UINT_TO_FP, DestVT, N)); } void SelectionDAGLowering::visitSIToFP(User &I){ // UIToFP is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::SINT_TO_FP, DestVT, N)); } void SelectionDAGLowering::visitPtrToInt(User &I) { // What to do depends on the size of the integer and the size of the pointer. // We can either truncate, zero extend, or no-op, accordingly. - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType SrcVT = N.getValueType(); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); - SDOperand Result; - if (MVT::getSizeInBits(DestVT) < MVT::getSizeInBits(SrcVT)) + SDValue N = getValue(I.getOperand(0)); + MVT SrcVT = N.getValueType(); + MVT DestVT = TLI.getValueType(I.getType()); + SDValue Result; + if (DestVT.bitsLT(SrcVT)) Result = DAG.getNode(ISD::TRUNCATE, DestVT, N); else // Note: ZERO_EXTEND can handle cases where the sizes are equal too @@ -2422,10 +2648,10 @@ void SelectionDAGLowering::visitPtrToInt(User &I) { void SelectionDAGLowering::visitIntToPtr(User &I) { // What to do depends on the size of the integer and the size of the pointer. // We can either truncate, zero extend, or no-op, accordingly. - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType SrcVT = N.getValueType(); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); - if (MVT::getSizeInBits(DestVT) < MVT::getSizeInBits(SrcVT)) + SDValue N = getValue(I.getOperand(0)); + MVT SrcVT = N.getValueType(); + MVT DestVT = TLI.getValueType(I.getType()); + if (DestVT.bitsLT(SrcVT)) setValue(&I, DAG.getNode(ISD::TRUNCATE, DestVT, N)); else // Note: ZERO_EXTEND can handle cases where the sizes are equal too @@ -2433,8 +2659,8 @@ void SelectionDAGLowering::visitIntToPtr(User &I) { } void SelectionDAGLowering::visitBitCast(User &I) { - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); // BitCast assures us that source and destination are the same size so this // is either a BIT_CONVERT or a no-op. @@ -2445,9 +2671,9 @@ void SelectionDAGLowering::visitBitCast(User &I) { } void SelectionDAGLowering::visitInsertElement(User &I) { - SDOperand InVec = getValue(I.getOperand(0)); - SDOperand InVal = getValue(I.getOperand(1)); - SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), + SDValue InVec = getValue(I.getOperand(0)); + SDValue InVal = getValue(I.getOperand(1)); + SDValue InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), getValue(I.getOperand(2))); setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, @@ -2456,26 +2682,92 @@ void SelectionDAGLowering::visitInsertElement(User &I) { } void SelectionDAGLowering::visitExtractElement(User &I) { - SDOperand InVec = getValue(I.getOperand(0)); - SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), + SDValue InVec = getValue(I.getOperand(0)); + SDValue InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), getValue(I.getOperand(1))); setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, TLI.getValueType(I.getType()), InVec, InIdx)); } void SelectionDAGLowering::visitShuffleVector(User &I) { - SDOperand V1 = getValue(I.getOperand(0)); - SDOperand V2 = getValue(I.getOperand(1)); - SDOperand Mask = getValue(I.getOperand(2)); + SDValue V1 = getValue(I.getOperand(0)); + SDValue V2 = getValue(I.getOperand(1)); + SDValue Mask = getValue(I.getOperand(2)); setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE, TLI.getValueType(I.getType()), V1, V2, Mask)); } +void SelectionDAGLowering::visitInsertValue(InsertValueInst &I) { + const Value *Op0 = I.getOperand(0); + const Value *Op1 = I.getOperand(1); + const Type *AggTy = I.getType(); + const Type *ValTy = Op1->getType(); + bool IntoUndef = isa(Op0); + bool FromUndef = isa(Op1); + + unsigned LinearIndex = ComputeLinearIndex(TLI, AggTy, + I.idx_begin(), I.idx_end()); + + SmallVector AggValueVTs; + ComputeValueVTs(TLI, AggTy, AggValueVTs); + SmallVector ValValueVTs; + ComputeValueVTs(TLI, ValTy, ValValueVTs); + + unsigned NumAggValues = AggValueVTs.size(); + unsigned NumValValues = ValValueVTs.size(); + SmallVector Values(NumAggValues); + + SDValue Agg = getValue(Op0); + SDValue Val = getValue(Op1); + unsigned i = 0; + // Copy the beginning value(s) from the original aggregate. + for (; i != LinearIndex; ++i) + Values[i] = IntoUndef ? DAG.getNode(ISD::UNDEF, AggValueVTs[i]) : + SDValue(Agg.Val, Agg.ResNo + i); + // Copy values from the inserted value(s). + for (; i != LinearIndex + NumValValues; ++i) + Values[i] = FromUndef ? DAG.getNode(ISD::UNDEF, AggValueVTs[i]) : + SDValue(Val.Val, Val.ResNo + i - LinearIndex); + // Copy remaining value(s) from the original aggregate. + for (; i != NumAggValues; ++i) + Values[i] = IntoUndef ? DAG.getNode(ISD::UNDEF, AggValueVTs[i]) : + SDValue(Agg.Val, Agg.ResNo + i); + + setValue(&I, DAG.getMergeValues(DAG.getVTList(&AggValueVTs[0], NumAggValues), + &Values[0], NumAggValues)); +} + +void SelectionDAGLowering::visitExtractValue(ExtractValueInst &I) { + const Value *Op0 = I.getOperand(0); + const Type *AggTy = Op0->getType(); + const Type *ValTy = I.getType(); + bool OutOfUndef = isa(Op0); + + unsigned LinearIndex = ComputeLinearIndex(TLI, AggTy, + I.idx_begin(), I.idx_end()); + + SmallVector ValValueVTs; + ComputeValueVTs(TLI, ValTy, ValValueVTs); + + unsigned NumValValues = ValValueVTs.size(); + SmallVector Values(NumValValues); + + SDValue Agg = getValue(Op0); + // Copy out the selected value(s). + for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i) + Values[i - LinearIndex] = + OutOfUndef ? DAG.getNode(ISD::UNDEF, Agg.Val->getValueType(Agg.ResNo + i)) : + SDValue(Agg.Val, Agg.ResNo + i); + + setValue(&I, DAG.getMergeValues(DAG.getVTList(&ValValueVTs[0], NumValValues), + &Values[0], NumValValues)); +} + void SelectionDAGLowering::visitGetElementPtr(User &I) { - SDOperand N = getValue(I.getOperand(0)); + SDValue N = getValue(I.getOperand(0)); const Type *Ty = I.getOperand(0)->getType(); for (GetElementPtrInst::op_iterator OI = I.op_begin()+1, E = I.op_end(); @@ -2505,13 +2797,13 @@ void SelectionDAGLowering::visitGetElementPtr(User &I) { // N = N + Idx * ElementSize; uint64_t ElementSize = TD->getABITypeSize(Ty); - SDOperand IdxN = getValue(Idx); + SDValue IdxN = getValue(Idx); // If the index is smaller or larger than intptr_t, truncate or extend // it. - if (IdxN.getValueType() < N.getValueType()) { + if (IdxN.getValueType().bitsLT(N.getValueType())) { IdxN = DAG.getNode(ISD::SIGN_EXTEND, N.getValueType(), IdxN); - } else if (IdxN.getValueType() > N.getValueType()) + } else if (IdxN.getValueType().bitsGT(N.getValueType())) IdxN = DAG.getNode(ISD::TRUNCATE, N.getValueType(), IdxN); // If this is a multiply by a power of two, turn it into a shl @@ -2524,7 +2816,7 @@ void SelectionDAGLowering::visitGetElementPtr(User &I) { continue; } - SDOperand Scale = DAG.getIntPtrConstant(ElementSize); + SDValue Scale = DAG.getIntPtrConstant(ElementSize); IdxN = DAG.getNode(ISD::MUL, N.getValueType(), IdxN, Scale); N = DAG.getNode(ISD::ADD, N.getValueType(), N, IdxN); } @@ -2544,11 +2836,11 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) { std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), I.getAlignment()); - SDOperand AllocSize = getValue(I.getArraySize()); - MVT::ValueType IntPtr = TLI.getPointerTy(); - if (IntPtr < AllocSize.getValueType()) + SDValue AllocSize = getValue(I.getArraySize()); + MVT IntPtr = TLI.getPointerTy(); + if (IntPtr.bitsLT(AllocSize.getValueType())) AllocSize = DAG.getNode(ISD::TRUNCATE, IntPtr, AllocSize); - else if (IntPtr > AllocSize.getValueType()) + else if (IntPtr.bitsGT(AllocSize.getValueType())) AllocSize = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, AllocSize); AllocSize = DAG.getNode(ISD::MUL, IntPtr, AllocSize, @@ -2570,10 +2862,10 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) { AllocSize = DAG.getNode(ISD::AND, AllocSize.getValueType(), AllocSize, DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1))); - SDOperand Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) }; - const MVT::ValueType *VTs = DAG.getNodeValueTypes(AllocSize.getValueType(), + SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) }; + const MVT *VTs = DAG.getNodeValueTypes(AllocSize.getValueType(), MVT::Other); - SDOperand DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, VTs, 2, Ops, 3); + SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, VTs, 2, Ops, 3); setValue(&I, DSA); DAG.setRoot(DSA.getValue(1)); @@ -2583,43 +2875,91 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) { } void SelectionDAGLowering::visitLoad(LoadInst &I) { - SDOperand Ptr = getValue(I.getOperand(0)); + const Value *SV = I.getOperand(0); + SDValue Ptr = getValue(SV); + + const Type *Ty = I.getType(); + bool isVolatile = I.isVolatile(); + unsigned Alignment = I.getAlignment(); + + SmallVector ValueVTs; + SmallVector Offsets; + ComputeValueVTs(TLI, Ty, ValueVTs, &Offsets); + unsigned NumValues = ValueVTs.size(); + if (NumValues == 0) + return; - SDOperand Root; + SDValue Root; + bool ConstantMemory = false; if (I.isVolatile()) + // Serialize volatile loads with other side effects. Root = getRoot(); - else { + else if (AA.pointsToConstantMemory(SV)) { + // Do not serialize (non-volatile) loads of constant memory with anything. + Root = DAG.getEntryNode(); + ConstantMemory = true; + } else { // Do not serialize non-volatile loads against each other. Root = DAG.getRoot(); } - setValue(&I, getLoadFrom(I.getType(), Ptr, I.getOperand(0), - Root, I.isVolatile(), I.getAlignment())); -} - -SDOperand SelectionDAGLowering::getLoadFrom(const Type *Ty, SDOperand Ptr, - const Value *SV, SDOperand Root, - bool isVolatile, - unsigned Alignment) { - SDOperand L = - DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, SV, 0, - isVolatile, Alignment); - - if (isVolatile) - DAG.setRoot(L.getValue(1)); - else - PendingLoads.push_back(L.getValue(1)); + SmallVector Values(NumValues); + SmallVector Chains(NumValues); + MVT PtrVT = Ptr.getValueType(); + for (unsigned i = 0; i != NumValues; ++i) { + SDValue L = DAG.getLoad(ValueVTs[i], Root, + DAG.getNode(ISD::ADD, PtrVT, Ptr, + DAG.getConstant(Offsets[i], PtrVT)), + SV, Offsets[i], + isVolatile, Alignment); + Values[i] = L; + Chains[i] = L.getValue(1); + } - return L; + if (!ConstantMemory) { + SDValue Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, + &Chains[0], NumValues); + if (isVolatile) + DAG.setRoot(Chain); + else + PendingLoads.push_back(Chain); + } + + setValue(&I, DAG.getMergeValues(DAG.getVTList(&ValueVTs[0], NumValues), + &Values[0], NumValues)); } void SelectionDAGLowering::visitStore(StoreInst &I) { Value *SrcV = I.getOperand(0); - SDOperand Src = getValue(SrcV); - SDOperand Ptr = getValue(I.getOperand(1)); - DAG.setRoot(DAG.getStore(getRoot(), Src, Ptr, I.getOperand(1), 0, - I.isVolatile(), I.getAlignment())); + Value *PtrV = I.getOperand(1); + + SmallVector ValueVTs; + SmallVector Offsets; + ComputeValueVTs(TLI, SrcV->getType(), ValueVTs, &Offsets); + unsigned NumValues = ValueVTs.size(); + if (NumValues == 0) + return; + + // Get the lowered operands. Note that we do this after + // checking if NumResults is zero, because with zero results + // the operands won't have values in the map. + SDValue Src = getValue(SrcV); + SDValue Ptr = getValue(PtrV); + + SDValue Root = getRoot(); + SmallVector Chains(NumValues); + MVT PtrVT = Ptr.getValueType(); + bool isVolatile = I.isVolatile(); + unsigned Alignment = I.getAlignment(); + for (unsigned i = 0; i != NumValues; ++i) + Chains[i] = DAG.getStore(Root, SDValue(Src.Val, Src.ResNo + i), + DAG.getNode(ISD::ADD, PtrVT, Ptr, + DAG.getConstant(Offsets[i], PtrVT)), + PtrV, Offsets[i], + isVolatile, Alignment); + + DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumValues)); } /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC @@ -2630,7 +2970,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, bool OnlyLoad = HasChain && I.onlyReadsMemory(); // Build the operand list. - SmallVector Ops; + SmallVector Ops; if (HasChain) { // If this intrinsic has side-effects, chainify it. if (OnlyLoad) { // We don't need to serialize loads against other loads. @@ -2645,20 +2985,20 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, // Add all operands of the call to the operand list. for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i) { - SDOperand Op = getValue(I.getOperand(i)); + SDValue Op = getValue(I.getOperand(i)); assert(TLI.isTypeLegal(Op.getValueType()) && "Intrinsic uses a non-legal type?"); Ops.push_back(Op); } - std::vector VTs; + std::vector VTs; if (I.getType() != Type::VoidTy) { - MVT::ValueType VT = TLI.getValueType(I.getType()); - if (MVT::isVector(VT)) { + MVT VT = TLI.getValueType(I.getType()); + if (VT.isVector()) { const VectorType *DestTy = cast(I.getType()); - MVT::ValueType EltVT = TLI.getValueType(DestTy->getElementType()); + MVT EltVT = TLI.getValueType(DestTy->getElementType()); - VT = MVT::getVectorType(EltVT, DestTy->getNumElements()); + VT = MVT::getVectorVT(EltVT, DestTy->getNumElements()); assert(VT != MVT::Other && "Intrinsic uses a non-legal type?"); } @@ -2668,10 +3008,10 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, if (HasChain) VTs.push_back(MVT::Other); - const MVT::ValueType *VTList = DAG.getNodeValueTypes(VTs); + const MVT *VTList = DAG.getNodeValueTypes(VTs); // Create the node. - SDOperand Result; + SDValue Result; if (!HasChain) Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VTList, VTs.size(), &Ops[0], Ops.size()); @@ -2683,7 +3023,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, &Ops[0], Ops.size()); if (HasChain) { - SDOperand Chain = Result.getValue(Result.Val->getNumValues()-1); + SDValue Chain = Result.getValue(Result.Val->getNumValues()-1); if (OnlyLoad) PendingLoads.push_back(Chain); else @@ -2691,7 +3031,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, } if (I.getType() != Type::VoidTy) { if (const VectorType *PTy = dyn_cast(I.getType())) { - MVT::ValueType VT = TLI.getValueType(PTy); + MVT VT = TLI.getValueType(PTy); Result = DAG.getNode(ISD::BIT_CONVERT, VT, Result); } setValue(&I, Result); @@ -2700,7 +3040,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, /// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V. static GlobalVariable *ExtractTypeInfo (Value *V) { - V = IntrinsicInst::StripPointerCasts(V); + V = V->stripPointerCasts(); GlobalVariable *GV = dyn_cast(V); assert ((GV || isa(V)) && "TypeInfo must be a global variable or NULL"); @@ -2761,6 +3101,22 @@ static void addCatchInfo(CallInst &I, MachineModuleInfo *MMI, } } + +/// Inlined utility function to implement binary input atomic intrinsics for +// visitIntrinsicCall: I is a call instruction +// Op is the associated NodeType for I +const char * +SelectionDAGLowering::implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op) { + SDValue Root = getRoot(); + SDValue L = DAG.getAtomic(Op, Root, + getValue(I.getOperand(1)), + getValue(I.getOperand(2)), + I.getOperand(1)); + setValue(&I, L); + DAG.setRoot(L.getValue(1)); + return 0; +} + /// visitIntrinsicCall - Lower the call to the specified intrinsic function. If /// we want to emit this as a call to a named external function, return the name /// otherwise lower it and return null. @@ -2790,9 +3146,9 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { break; case Intrinsic::memcpy_i32: case Intrinsic::memcpy_i64: { - SDOperand Op1 = getValue(I.getOperand(1)); - SDOperand Op2 = getValue(I.getOperand(2)); - SDOperand Op3 = getValue(I.getOperand(3)); + SDValue Op1 = getValue(I.getOperand(1)); + SDValue Op2 = getValue(I.getOperand(2)); + SDValue Op3 = getValue(I.getOperand(3)); unsigned Align = cast(I.getOperand(4))->getZExtValue(); DAG.setRoot(DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Align, false, I.getOperand(1), 0, I.getOperand(2), 0)); @@ -2800,9 +3156,9 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::memset_i32: case Intrinsic::memset_i64: { - SDOperand Op1 = getValue(I.getOperand(1)); - SDOperand Op2 = getValue(I.getOperand(2)); - SDOperand Op3 = getValue(I.getOperand(3)); + SDValue Op1 = getValue(I.getOperand(1)); + SDValue Op2 = getValue(I.getOperand(2)); + SDValue Op3 = getValue(I.getOperand(3)); unsigned Align = cast(I.getOperand(4))->getZExtValue(); DAG.setRoot(DAG.getMemset(getRoot(), Op1, Op2, Op3, Align, I.getOperand(1), 0)); @@ -2810,9 +3166,9 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::memmove_i32: case Intrinsic::memmove_i64: { - SDOperand Op1 = getValue(I.getOperand(1)); - SDOperand Op2 = getValue(I.getOperand(2)); - SDOperand Op3 = getValue(I.getOperand(3)); + SDValue Op1 = getValue(I.getOperand(1)); + SDValue Op2 = getValue(I.getOperand(2)); + SDValue Op3 = getValue(I.getOperand(3)); unsigned Align = cast(I.getOperand(4))->getZExtValue(); // If the source and destination are known to not be aliases, we can @@ -2835,20 +3191,12 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); DbgStopPointInst &SPI = cast(I); if (MMI && SPI.getContext() && MMI->Verify(SPI.getContext())) { - SDOperand Ops[5]; - - Ops[0] = getRoot(); - Ops[1] = getValue(SPI.getLineValue()); - Ops[2] = getValue(SPI.getColumnValue()); - DebugInfoDesc *DD = MMI->getDescFor(SPI.getContext()); assert(DD && "Not a debug information descriptor"); - CompileUnitDesc *CompileUnit = cast(DD); - - Ops[3] = DAG.getString(CompileUnit->getFileName()); - Ops[4] = DAG.getString(CompileUnit->getDirectory()); - - DAG.setRoot(DAG.getNode(ISD::LOCATION, MVT::Other, Ops, 5)); + DAG.setRoot(DAG.getDbgStopPoint(getRoot(), + SPI.getLine(), + SPI.getColumn(), + cast(DD))); } return 0; @@ -2858,9 +3206,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DbgRegionStartInst &RSI = cast(I); if (MMI && RSI.getContext() && MMI->Verify(RSI.getContext())) { unsigned LabelID = MMI->RecordRegionStart(RSI.getContext()); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), - DAG.getConstant(LabelID, MVT::i32), - DAG.getConstant(0, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::DBG_LABEL, getRoot(), LabelID)); } return 0; @@ -2870,9 +3216,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DbgRegionEndInst &REI = cast(I); if (MMI && REI.getContext() && MMI->Verify(REI.getContext())) { unsigned LabelID = MMI->RecordRegionEnd(REI.getContext()); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), - DAG.getConstant(LabelID, MVT::i32), - DAG.getConstant(0, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::DBG_LABEL, getRoot(), LabelID)); } return 0; @@ -2889,8 +3233,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { assert(DD && "Not a debug information descriptor"); SubprogramDesc *Subprogram = cast(DD); const CompileUnitDesc *CompileUnit = Subprogram->getFile(); - unsigned SrcFile = MMI->RecordSource(CompileUnit->getDirectory(), - CompileUnit->getFileName()); + unsigned SrcFile = MMI->RecordSource(CompileUnit); // Record the source line but does create a label. It will be emitted // at asm emission time. MMI->RecordSourceLine(Subprogram->getLine(), 0, SrcFile); @@ -2916,9 +3259,9 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } // Insert the EXCEPTIONADDR instruction. SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other); - SDOperand Ops[1]; + SDValue Ops[1]; Ops[0] = DAG.getRoot(); - SDOperand Op = DAG.getNode(ISD::EXCEPTIONADDR, VTs, Ops, 1); + SDValue Op = DAG.getNode(ISD::EXCEPTIONADDR, VTs, Ops, 1); setValue(&I, Op); DAG.setRoot(Op.getValue(1)); return 0; @@ -2927,7 +3270,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::eh_selector_i32: case Intrinsic::eh_selector_i64: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - MVT::ValueType VT = (Intrinsic == Intrinsic::eh_selector_i32 ? + MVT VT = (Intrinsic == Intrinsic::eh_selector_i32 ? MVT::i32 : MVT::i64); if (MMI) { @@ -2944,10 +3287,10 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { // Insert the EHSELECTION instruction. SDVTList VTs = DAG.getVTList(VT, MVT::Other); - SDOperand Ops[2]; + SDValue Ops[2]; Ops[0] = getValue(I.getOperand(1)); Ops[1] = getRoot(); - SDOperand Op = DAG.getNode(ISD::EHSELECTION, VTs, Ops, 2); + SDValue Op = DAG.getNode(ISD::EHSELECTION, VTs, Ops, 2); setValue(&I, Op); DAG.setRoot(Op.getValue(1)); } else { @@ -2960,7 +3303,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::eh_typeid_for_i32: case Intrinsic::eh_typeid_for_i64: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - MVT::ValueType VT = (Intrinsic == Intrinsic::eh_typeid_for_i32 ? + MVT VT = (Intrinsic == Intrinsic::eh_typeid_for_i32 ? MVT::i32 : MVT::i64); if (MMI) { @@ -3003,16 +3346,16 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::eh_dwarf_cfa: { - MVT::ValueType VT = getValue(I.getOperand(1)).getValueType(); - SDOperand CfaArg; - if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy())) + MVT VT = getValue(I.getOperand(1)).getValueType(); + SDValue CfaArg; + if (VT.bitsGT(TLI.getPointerTy())) CfaArg = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), getValue(I.getOperand(1))); else CfaArg = DAG.getNode(ISD::SIGN_EXTEND, TLI.getPointerTy(), getValue(I.getOperand(1))); - SDOperand Offset = DAG.getNode(ISD::ADD, + SDValue Offset = DAG.getNode(ISD::ADD, TLI.getPointerTy(), DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, TLI.getPointerTy()), @@ -3055,13 +3398,13 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { getValue(I.getOperand(2)))); return 0; case Intrinsic::pcmarker: { - SDOperand Tmp = getValue(I.getOperand(1)); + SDValue Tmp = getValue(I.getOperand(1)); DAG.setRoot(DAG.getNode(ISD::PCMARKER, MVT::Other, getRoot(), Tmp)); return 0; } case Intrinsic::readcyclecounter: { - SDOperand Op = getRoot(); - SDOperand Tmp = DAG.getNode(ISD::READCYCLECOUNTER, + SDValue Op = getRoot(); + SDValue Tmp = DAG.getNode(ISD::READCYCLECOUNTER, DAG.getNodeValueTypes(MVT::i64, MVT::Other), 2, &Op, 1); setValue(&I, Tmp); @@ -3084,36 +3427,36 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { getValue(I.getOperand(1)))); return 0; case Intrinsic::cttz: { - SDOperand Arg = getValue(I.getOperand(1)); - MVT::ValueType Ty = Arg.getValueType(); - SDOperand result = DAG.getNode(ISD::CTTZ, Ty, Arg); + SDValue Arg = getValue(I.getOperand(1)); + MVT Ty = Arg.getValueType(); + SDValue result = DAG.getNode(ISD::CTTZ, Ty, Arg); setValue(&I, result); return 0; } case Intrinsic::ctlz: { - SDOperand Arg = getValue(I.getOperand(1)); - MVT::ValueType Ty = Arg.getValueType(); - SDOperand result = DAG.getNode(ISD::CTLZ, Ty, Arg); + SDValue Arg = getValue(I.getOperand(1)); + MVT Ty = Arg.getValueType(); + SDValue result = DAG.getNode(ISD::CTLZ, Ty, Arg); setValue(&I, result); return 0; } case Intrinsic::ctpop: { - SDOperand Arg = getValue(I.getOperand(1)); - MVT::ValueType Ty = Arg.getValueType(); - SDOperand result = DAG.getNode(ISD::CTPOP, Ty, Arg); + SDValue Arg = getValue(I.getOperand(1)); + MVT Ty = Arg.getValueType(); + SDValue result = DAG.getNode(ISD::CTPOP, Ty, Arg); setValue(&I, result); return 0; } case Intrinsic::stacksave: { - SDOperand Op = getRoot(); - SDOperand Tmp = DAG.getNode(ISD::STACKSAVE, + SDValue Op = getRoot(); + SDValue Tmp = DAG.getNode(ISD::STACKSAVE, DAG.getNodeValueTypes(TLI.getPointerTy(), MVT::Other), 2, &Op, 1); setValue(&I, Tmp); DAG.setRoot(Tmp.getValue(1)); return 0; } case Intrinsic::stackrestore: { - SDOperand Tmp = getValue(I.getOperand(1)); + SDValue Tmp = getValue(I.getOperand(1)); DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, MVT::Other, getRoot(), Tmp)); return 0; } @@ -3122,10 +3465,9 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return 0; case Intrinsic::init_trampoline: { - const Function *F = - cast(IntrinsicInst::StripPointerCasts(I.getOperand(2))); + const Function *F = cast(I.getOperand(2)->stripPointerCasts()); - SDOperand Ops[6]; + SDValue Ops[6]; Ops[0] = getRoot(); Ops[1] = getValue(I.getOperand(1)); Ops[2] = getValue(I.getOperand(2)); @@ -3133,7 +3475,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { Ops[4] = DAG.getSrcValue(I.getOperand(1)); Ops[5] = DAG.getSrcValue(F); - SDOperand Tmp = DAG.getNode(ISD::TRAMPOLINE, + SDValue Tmp = DAG.getNode(ISD::TRAMPOLINE, DAG.getNodeValueTypes(TLI.getPointerTy(), MVT::Other), 2, Ops, 6); @@ -3168,7 +3510,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return 0; } case Intrinsic::prefetch: { - SDOperand Ops[4]; + SDValue Ops[4]; Ops[0] = getRoot(); Ops[1] = getValue(I.getOperand(1)); Ops[2] = getValue(I.getOperand(2)); @@ -3178,7 +3520,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::memory_barrier: { - SDOperand Ops[6]; + SDValue Ops[6]; Ops[0] = getRoot(); for (int x = 1; x < 6; ++x) Ops[x] = getValue(I.getOperand(x)); @@ -3186,43 +3528,44 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DAG.setRoot(DAG.getNode(ISD::MEMBARRIER, MVT::Other, &Ops[0], 6)); return 0; } - case Intrinsic::atomic_lcs: { - SDOperand Root = getRoot(); - SDOperand O3 = getValue(I.getOperand(3)); - SDOperand L = DAG.getAtomic(ISD::ATOMIC_LCS, Root, + case Intrinsic::atomic_cmp_swap: { + SDValue Root = getRoot(); + SDValue L = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, Root, getValue(I.getOperand(1)), getValue(I.getOperand(2)), - O3, O3.getValueType()); + getValue(I.getOperand(3)), + I.getOperand(1)); setValue(&I, L); DAG.setRoot(L.getValue(1)); return 0; } - case Intrinsic::atomic_las: { - SDOperand Root = getRoot(); - SDOperand O2 = getValue(I.getOperand(2)); - SDOperand L = DAG.getAtomic(ISD::ATOMIC_LAS, Root, - getValue(I.getOperand(1)), - O2, O2.getValueType()); - setValue(&I, L); - DAG.setRoot(L.getValue(1)); - return 0; - } - case Intrinsic::atomic_swap: { - SDOperand Root = getRoot(); - SDOperand O2 = getValue(I.getOperand(2)); - SDOperand L = DAG.getAtomic(ISD::ATOMIC_SWAP, Root, - getValue(I.getOperand(1)), - O2, O2.getValueType()); - setValue(&I, L); - DAG.setRoot(L.getValue(1)); - return 0; - } - + case Intrinsic::atomic_load_add: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_ADD); + case Intrinsic::atomic_load_sub: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_SUB); + case Intrinsic::atomic_load_and: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_AND); + case Intrinsic::atomic_load_or: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_OR); + case Intrinsic::atomic_load_xor: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_XOR); + case Intrinsic::atomic_load_nand: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_NAND); + case Intrinsic::atomic_load_min: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MIN); + case Intrinsic::atomic_load_max: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MAX); + case Intrinsic::atomic_load_umin: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMIN); + case Intrinsic::atomic_load_umax: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMAX); + case Intrinsic::atomic_swap: + return implVisitBinaryAtomic(I, ISD::ATOMIC_SWAP); } } -void SelectionDAGLowering::LowerCallTo(CallSite CS, SDOperand Callee, +void SelectionDAGLowering::LowerCallTo(CallSite CS, SDValue Callee, bool IsTailCall, MachineBasicBlock *LandingPad) { const PointerType *PT = cast(CS.getCalledValue()->getType()); @@ -3235,7 +3578,7 @@ void SelectionDAGLowering::LowerCallTo(CallSite CS, SDOperand Callee, Args.reserve(CS.arg_size()); for (CallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); i != e; ++i) { - SDOperand ArgNode = getValue(*i); + SDValue ArgNode = getValue(*i); Entry.Node = ArgNode; Entry.Ty = (*i)->getType(); unsigned attrInd = i - CS.arg_begin() + 1; @@ -3256,12 +3599,10 @@ void SelectionDAGLowering::LowerCallTo(CallSite CS, SDOperand Callee, // Both PendingLoads and PendingExports must be flushed here; // this call might not return. (void)getRoot(); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getControlRoot(), - DAG.getConstant(BeginLabel, MVT::i32), - DAG.getConstant(1, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::EH_LABEL, getControlRoot(), BeginLabel)); } - std::pair Result = + std::pair Result = TLI.LowerCallTo(getRoot(), CS.getType(), CS.paramHasAttr(0, ParamAttr::SExt), CS.paramHasAttr(0, ParamAttr::ZExt), @@ -3275,9 +3616,7 @@ void SelectionDAGLowering::LowerCallTo(CallSite CS, SDOperand Callee, // Insert a label at the end of the invoke call to mark the try range. This // can be used to detect deletion of the invoke via the MachineModuleInfo. EndLabel = MMI->NextLabelID(); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), - DAG.getConstant(EndLabel, MVT::i32), - DAG.getConstant(1, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::EH_LABEL, getRoot(), EndLabel)); // Inform MachineModuleInfo of range. MMI->addInvoke(LandingPad, BeginLabel, EndLabel); @@ -3308,8 +3647,8 @@ void SelectionDAGLowering::visitCall(CallInst &I) { I.getOperand(1)->getType()->isFloatingPoint() && I.getType() == I.getOperand(1)->getType() && I.getType() == I.getOperand(2)->getType()) { - SDOperand LHS = getValue(I.getOperand(1)); - SDOperand RHS = getValue(I.getOperand(2)); + SDValue LHS = getValue(I.getOperand(1)); + SDValue RHS = getValue(I.getOperand(2)); setValue(&I, DAG.getNode(ISD::FCOPYSIGN, LHS.getValueType(), LHS, RHS)); return; @@ -3321,7 +3660,7 @@ void SelectionDAGLowering::visitCall(CallInst &I) { if (I.getNumOperands() == 2 && // Basic sanity checks. I.getOperand(1)->getType()->isFloatingPoint() && I.getType() == I.getOperand(1)->getType()) { - SDOperand Tmp = getValue(I.getOperand(1)); + SDValue Tmp = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FABS, Tmp.getValueType(), Tmp)); return; } @@ -3332,7 +3671,7 @@ void SelectionDAGLowering::visitCall(CallInst &I) { if (I.getNumOperands() == 2 && // Basic sanity checks. I.getOperand(1)->getType()->isFloatingPoint() && I.getType() == I.getOperand(1)->getType()) { - SDOperand Tmp = getValue(I.getOperand(1)); + SDValue Tmp = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FSIN, Tmp.getValueType(), Tmp)); return; } @@ -3343,7 +3682,7 @@ void SelectionDAGLowering::visitCall(CallInst &I) { if (I.getNumOperands() == 2 && // Basic sanity checks. I.getOperand(1)->getType()->isFloatingPoint() && I.getType() == I.getOperand(1)->getType()) { - SDOperand Tmp = getValue(I.getOperand(1)); + SDValue Tmp = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FCOS, Tmp.getValueType(), Tmp)); return; } @@ -3354,7 +3693,7 @@ void SelectionDAGLowering::visitCall(CallInst &I) { return; } - SDOperand Callee; + SDValue Callee; if (!RenameFn) Callee = getValue(I.getOperand(0)); else @@ -3364,46 +3703,75 @@ void SelectionDAGLowering::visitCall(CallInst &I) { } -void SelectionDAGLowering::visitGetResult(GetResultInst &I) { - if (isa(I.getOperand(0))) { - SDOperand Undef = DAG.getNode(ISD::UNDEF, TLI.getValueType(I.getType())); - setValue(&I, Undef); - } else { - SDOperand Call = getValue(I.getOperand(0)); - - // To add support for individual return values with aggregate types, - // we'd need a way to take a getresult index and determine which - // values of the Call SDNode are associated with it. - assert(TLI.getValueType(I.getType(), true) != MVT::Other && - "Individual return values must not be aggregates!"); - - setValue(&I, SDOperand(Call.Val, I.getIndex())); - } -} - - /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from /// this value and returns the result as a ValueVT value. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. -SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG, - SDOperand &Chain, SDOperand *Flag)const{ +SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, + SDValue &Chain, + SDValue *Flag) const { // Assemble the legal parts into the final values. - SmallVector Values(ValueVTs.size()); - for (unsigned Value = 0, Part = 0; Value != ValueVTs.size(); ++Value) { + SmallVector Values(ValueVTs.size()); + SmallVector Parts; + for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { // Copy the legal parts from the registers. - MVT::ValueType ValueVT = ValueVTs[Value]; + MVT ValueVT = ValueVTs[Value]; unsigned NumRegs = TLI->getNumRegisters(ValueVT); - MVT::ValueType RegisterVT = RegVTs[Value]; + MVT RegisterVT = RegVTs[Value]; - SmallVector Parts(NumRegs); + Parts.resize(NumRegs); for (unsigned i = 0; i != NumRegs; ++i) { - SDOperand P = Flag ? - DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT, *Flag) : - DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT); - Chain = P.getValue(1); - if (Flag) + SDValue P; + if (Flag == 0) + P = DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT); + else { + P = DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT, *Flag); *Flag = P.getValue(2); + } + Chain = P.getValue(1); + + // If the source register was virtual and if we know something about it, + // add an assert node. + if (TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) && + RegisterVT.isInteger() && !RegisterVT.isVector()) { + unsigned SlotNo = Regs[Part+i]-TargetRegisterInfo::FirstVirtualRegister; + FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); + if (FLI.LiveOutRegInfo.size() > SlotNo) { + FunctionLoweringInfo::LiveOutInfo &LOI = FLI.LiveOutRegInfo[SlotNo]; + + unsigned RegSize = RegisterVT.getSizeInBits(); + unsigned NumSignBits = LOI.NumSignBits; + unsigned NumZeroBits = LOI.KnownZero.countLeadingOnes(); + + // FIXME: We capture more information than the dag can represent. For + // now, just use the tightest assertzext/assertsext possible. + bool isSExt = true; + MVT FromVT(MVT::Other); + if (NumSignBits == RegSize) + isSExt = true, FromVT = MVT::i1; // ASSERT SEXT 1 + else if (NumZeroBits >= RegSize-1) + isSExt = false, FromVT = MVT::i1; // ASSERT ZEXT 1 + else if (NumSignBits > RegSize-8) + isSExt = true, FromVT = MVT::i8; // ASSERT SEXT 8 + else if (NumZeroBits >= RegSize-9) + isSExt = false, FromVT = MVT::i8; // ASSERT ZEXT 8 + else if (NumSignBits > RegSize-16) + isSExt = true, FromVT = MVT::i16; // ASSERT SEXT 16 + else if (NumZeroBits >= RegSize-17) + isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16 + else if (NumSignBits > RegSize-32) + isSExt = true, FromVT = MVT::i32; // ASSERT SEXT 32 + else if (NumZeroBits >= RegSize-33) + isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32 + + if (FromVT != MVT::Other) { + P = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, + RegisterVT, P, DAG.getValueType(FromVT)); + + } + } + } + Parts[Part+i] = P; } @@ -3411,24 +3779,24 @@ SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG, ValueVT); Part += NumRegs; } - return DAG.getNode(ISD::MERGE_VALUES, - DAG.getVTList(&ValueVTs[0], ValueVTs.size()), - &Values[0], ValueVTs.size()); + + return DAG.getMergeValues(DAG.getVTList(&ValueVTs[0], ValueVTs.size()), + &Values[0], ValueVTs.size()); } /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the /// specified value into the registers specified by this object. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. -void RegsForValue::getCopyToRegs(SDOperand Val, SelectionDAG &DAG, - SDOperand &Chain, SDOperand *Flag) const { +void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, + SDValue &Chain, SDValue *Flag) const { // Get the list of the values's legal parts. unsigned NumRegs = Regs.size(); - SmallVector Parts(NumRegs); - for (unsigned Value = 0, Part = 0; Value != ValueVTs.size(); ++Value) { - MVT::ValueType ValueVT = ValueVTs[Value]; + SmallVector Parts(NumRegs); + for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { + MVT ValueVT = ValueVTs[Value]; unsigned NumParts = TLI->getNumRegisters(ValueVT); - MVT::ValueType RegisterVT = RegVTs[Value]; + MVT RegisterVT = RegVTs[Value]; getCopyToParts(DAG, Val.getValue(Val.ResNo + Value), &Parts[Part], NumParts, RegisterVT); @@ -3436,34 +3804,46 @@ void RegsForValue::getCopyToRegs(SDOperand Val, SelectionDAG &DAG, } // Copy the parts into the registers. - SmallVector Chains(NumRegs); + SmallVector Chains(NumRegs); for (unsigned i = 0; i != NumRegs; ++i) { - SDOperand Part = Flag ? - DAG.getCopyToReg(Chain, Regs[i], Parts[i], *Flag) : - DAG.getCopyToReg(Chain, Regs[i], Parts[i]); - Chains[i] = Part.getValue(0); - if (Flag) + SDValue Part; + if (Flag == 0) + Part = DAG.getCopyToReg(Chain, Regs[i], Parts[i]); + else { + Part = DAG.getCopyToReg(Chain, Regs[i], Parts[i], *Flag); *Flag = Part.getValue(1); + } + Chains[i] = Part.getValue(0); } - Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumRegs); + + if (NumRegs == 1 || Flag) + // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is + // flagged to it. That is the CopyToReg nodes and the user are considered + // a single scheduling unit. If we create a TokenFactor and return it as + // chain, then the TokenFactor is both a predecessor (operand) of the + // user as well as a successor (the TF operands are flagged to the user). + // c1, f1 = CopyToReg + // c2, f2 = CopyToReg + // c3 = TokenFactor c1, c2 + // ... + // = op c3, ..., f2 + Chain = Chains[NumRegs-1]; + else + Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumRegs); } /// AddInlineAsmOperands - Add this value to the specified inlineasm node /// operand list. This adds the code marker and includes the number of /// values added into it. void RegsForValue::AddInlineAsmOperands(unsigned Code, SelectionDAG &DAG, - std::vector &Ops) const { - MVT::ValueType IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); + std::vector &Ops) const { + MVT IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); Ops.push_back(DAG.getTargetConstant(Code | (Regs.size() << 3), IntPtrTy)); - for (unsigned Value = 0, Reg = 0; Value != ValueVTs.size(); ++Value) { - MVT::ValueType ValueVT = ValueVTs[Value]; - unsigned NumRegs = TLI->getNumRegisters(ValueVT); - MVT::ValueType RegisterVT = RegVTs[Value]; - for (unsigned i = 0; i != NumRegs; ++i) { - SDOperand RegOp = DAG.getRegister(Regs[Reg+i], RegisterVT); - Ops.push_back(RegOp); - } - Reg += NumRegs; + for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { + unsigned NumRegs = TLI->getNumRegisters(ValueVTs[Value]); + MVT RegisterVT = RegVTs[Value]; + for (unsigned i = 0; i != NumRegs; ++i) + Ops.push_back(DAG.getRegister(Regs[Reg++], RegisterVT)); } } @@ -3474,11 +3854,11 @@ static const TargetRegisterClass * isAllocatableRegister(unsigned Reg, MachineFunction &MF, const TargetLowering &TLI, const TargetRegisterInfo *TRI) { - MVT::ValueType FoundVT = MVT::Other; + MVT FoundVT = MVT::Other; const TargetRegisterClass *FoundRC = 0; for (TargetRegisterInfo::regclass_iterator RCI = TRI->regclass_begin(), E = TRI->regclass_end(); RCI != E; ++RCI) { - MVT::ValueType ThisVT = MVT::Other; + MVT ThisVT = MVT::Other; const TargetRegisterClass *RC = *RCI; // If none of the the value types for this register class are valid, we @@ -3489,8 +3869,7 @@ isAllocatableRegister(unsigned Reg, MachineFunction &MF, // If we have already found this register in a different register class, // choose the one with the largest VT specified. For example, on // PowerPC, we favor f64 register classes over f32. - if (FoundVT == MVT::Other || - MVT::getSizeInBits(FoundVT) < MVT::getSizeInBits(*I)) { + if (FoundVT == MVT::Other || FoundVT.bitsLT(*I)) { ThisVT = *I; break; } @@ -3524,7 +3903,7 @@ struct SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { /// CallOperand - If this is the result output operand or a clobber /// this is null, otherwise it is the incoming operand to the CallInst. /// This gets modified as the asm is processed. - SDOperand CallOperand; + SDValue CallOperand; /// AssignedRegs - If this is a register or register class operand, this /// contains the set of register corresponding to the operand. @@ -3616,8 +3995,8 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, bool HasEarlyClobber, unsigned NumRegs = 1; if (OpInfo.ConstraintVT != MVT::Other) NumRegs = TLI.getNumRegisters(OpInfo.ConstraintVT); - MVT::ValueType RegVT; - MVT::ValueType ValueVT = OpInfo.ConstraintVT; + MVT RegVT; + MVT ValueVT = OpInfo.ConstraintVT; // If this is a constraint for a specific physical register, like {r17}, @@ -3637,14 +4016,13 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, bool HasEarlyClobber, // If this is an expanded reference, add the rest of the regs to Regs. if (NumRegs != 1) { TargetRegisterClass::iterator I = PhysReg.second->begin(); - TargetRegisterClass::iterator E = PhysReg.second->end(); for (; *I != PhysReg.first; ++I) - assert(I != E && "Didn't find reg!"); + assert(I != PhysReg.second->end() && "Didn't find reg!"); // Already added the first reg. --NumRegs; ++I; for (; NumRegs; --NumRegs, ++I) { - assert(I != E && "Ran out of registers to allocate!"); + assert(I != PhysReg.second->end() && "Ran out of registers to allocate!"); Regs.push_back(*I); } } @@ -3735,7 +4113,6 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, bool HasEarlyClobber, } // Otherwise, we couldn't allocate enough registers for this. - return; } @@ -3747,8 +4124,8 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { /// ConstraintOperands - Information about all of the constraints. std::vector ConstraintOperands; - SDOperand Chain = getRoot(); - SDOperand Flag; + SDValue Chain = getRoot(); + SDValue Flag; std::set OutputRegs, InputRegs; @@ -3768,7 +4145,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { ConstraintOperands.push_back(SDISelAsmOperandInfo(ConstraintInfos[i])); SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back(); - MVT::ValueType OpVT = MVT::Other; + MVT OpVT = MVT::Other; // Compute the value type for each operand. switch (OpInfo.Type) { @@ -3810,9 +4187,9 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { if (OpInfo.isIndirect) OpTy = cast(OpTy)->getElementType(); - // If OpTy is not a first-class value, it may be a struct/union that we + // If OpTy is not a single value, it may be a struct/union that we // can tile with integers. - if (!OpTy->isFirstClassType() && OpTy->isSized()) { + if (!OpTy->isSingleValueType() && OpTy->isSized()) { unsigned BitSize = TD->getTypeSizeInBits(OpTy); switch (BitSize) { default: break; @@ -3879,7 +4256,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { unsigned Align = TLI.getTargetData()->getPrefTypeAlignment(Ty); MachineFunction &MF = DAG.getMachineFunction(); int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align); - SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy()); + SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy()); Chain = DAG.getStore(Chain, OpInfo.CallOperand, StackSlot, NULL, 0); OpInfo.CallOperand = StackSlot; } @@ -3910,8 +4287,8 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { } // AsmNodeOperands - The operands for the ISD::INLINEASM node. - std::vector AsmNodeOperands; - AsmNodeOperands.push_back(SDOperand()); // reserve space for input chain + std::vector AsmNodeOperands; + AsmNodeOperands.push_back(SDValue()); // reserve space for input chain AsmNodeOperands.push_back( DAG.getTargetExternalSymbol(IA->getAsmString().c_str(), MVT::Other)); @@ -3919,7 +4296,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // Loop over all of the inputs, copying the operand values into the // appropriate registers and processing the output regs. RegsForValue RetValRegs; - + // IndirectStoresToEmit - The set of stores to emit after the inline asm node. std::vector > IndirectStoresToEmit; @@ -3946,20 +4323,21 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // Copy the output from the appropriate register. Find a register that // we can use. if (OpInfo.AssignedRegs.Regs.empty()) { - cerr << "Couldn't allocate output reg for contraint '" + cerr << "Couldn't allocate output reg for constraint '" << OpInfo.ConstraintCode << "'!\n"; exit(1); } - if (!OpInfo.isIndirect) { - // This is the result value of the call. - assert(RetValRegs.Regs.empty() && - "Cannot have multiple output constraints yet!"); - assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); - RetValRegs = OpInfo.AssignedRegs; - } else { + // If this is an indirect operand, store through the pointer after the + // asm. + if (OpInfo.isIndirect) { IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs, OpInfo.CallOperandVal)); + } else { + // This is the result value of the call. + assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); + // Concatenate this output onto the outputs list. + RetValRegs.append(OpInfo.AssignedRegs); } // Add information to the INLINEASM node to know that this register is @@ -3969,7 +4347,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { break; } case InlineAsm::isInput: { - SDOperand InOperandVal = OpInfo.CallOperand; + SDValue InOperandVal = OpInfo.CallOperand; if (isdigit(OpInfo.ConstraintCode[0])) { // Matching constraint? // If this is required to match an output register we have already set, @@ -3995,8 +4373,8 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // Add NumOps>>3 registers to MatchedRegs. RegsForValue MatchedRegs; MatchedRegs.TLI = &TLI; - MatchedRegs.ValueVTs.resize(1, InOperandVal.getValueType()); - MatchedRegs.RegVTs.resize(1, AsmNodeOperands[CurOp+1].getValueType()); + MatchedRegs.ValueVTs.push_back(InOperandVal.getValueType()); + MatchedRegs.RegVTs.push_back(AsmNodeOperands[CurOp+1].getValueType()); for (unsigned i = 0, e = NumOps>>3; i != e; ++i) { unsigned Reg = cast(AsmNodeOperands[++CurOp])->getReg(); @@ -4023,7 +4401,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { assert(!OpInfo.isIndirect && "Don't know how to handle indirect other inputs yet!"); - std::vector Ops; + std::vector Ops; TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode[0], Ops, DAG); if (Ops.empty()) { @@ -4090,34 +4468,41 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // If this asm returns a register value, copy the result from that register // and set it as the value of the call. if (!RetValRegs.Regs.empty()) { - SDOperand Val = RetValRegs.getCopyFromRegs(DAG, Chain, &Flag); - - // If the result of the inline asm is a vector, it may have the wrong - // width/num elts. Make sure to convert it to the right type with + SDValue Val = RetValRegs.getCopyFromRegs(DAG, Chain, &Flag); + + // If any of the results of the inline asm is a vector, it may have the + // wrong width/num elts. This can happen for register classes that can + // contain multiple different value types. The preg or vreg allocated may + // not have the same VT as was expected. Convert it to the right type with // bit_convert. - if (MVT::isVector(Val.getValueType())) { - const VectorType *VTy = cast(CS.getType()); - MVT::ValueType DesiredVT = TLI.getValueType(VTy); - - Val = DAG.getNode(ISD::BIT_CONVERT, DesiredVT, Val); + if (const StructType *ResSTy = dyn_cast(CS.getType())) { + for (unsigned i = 0, e = ResSTy->getNumElements(); i != e; ++i) { + if (Val.Val->getValueType(i).isVector()) + Val = DAG.getNode(ISD::BIT_CONVERT, + TLI.getValueType(ResSTy->getElementType(i)), Val); + } + } else { + if (Val.getValueType().isVector()) + Val = DAG.getNode(ISD::BIT_CONVERT, TLI.getValueType(CS.getType()), + Val); } - + setValue(CS.getInstruction(), Val); } - std::vector > StoresToEmit; + std::vector > StoresToEmit; // Process indirect outputs, first output all of the flagged copies out of // physregs. for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) { RegsForValue &OutRegs = IndirectStoresToEmit[i].first; Value *Ptr = IndirectStoresToEmit[i].second; - SDOperand OutVal = OutRegs.getCopyFromRegs(DAG, Chain, &Flag); + SDValue OutVal = OutRegs.getCopyFromRegs(DAG, Chain, &Flag); StoresToEmit.push_back(std::make_pair(OutVal, Ptr)); } // Emit the non-flagged stores from the physregs. - SmallVector OutChains; + SmallVector OutChains; for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) OutChains.push_back(DAG.getStore(Chain, StoresToEmit[i].first, getValue(StoresToEmit[i].second), @@ -4130,13 +4515,13 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { void SelectionDAGLowering::visitMalloc(MallocInst &I) { - SDOperand Src = getValue(I.getOperand(0)); + SDValue Src = getValue(I.getOperand(0)); - MVT::ValueType IntPtr = TLI.getPointerTy(); + MVT IntPtr = TLI.getPointerTy(); - if (IntPtr < Src.getValueType()) + if (IntPtr.bitsLT(Src.getValueType())) Src = DAG.getNode(ISD::TRUNCATE, IntPtr, Src); - else if (IntPtr > Src.getValueType()) + else if (IntPtr.bitsGT(Src.getValueType())) Src = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, Src); // Scale the source by the type size. @@ -4150,7 +4535,7 @@ void SelectionDAGLowering::visitMalloc(MallocInst &I) { Entry.Ty = TLI.getTargetData()->getIntPtrType(); Args.push_back(Entry); - std::pair Result = + std::pair Result = TLI.LowerCallTo(getRoot(), I.getType(), false, false, false, CallingConv::C, true, DAG.getExternalSymbol("malloc", IntPtr), Args, DAG); setValue(&I, Result.first); // Pointers always fit in registers @@ -4163,8 +4548,8 @@ void SelectionDAGLowering::visitFree(FreeInst &I) { Entry.Node = getValue(I.getOperand(0)); Entry.Ty = TLI.getTargetData()->getIntPtrType(); Args.push_back(Entry); - MVT::ValueType IntPtr = TLI.getPointerTy(); - std::pair Result = + MVT IntPtr = TLI.getPointerTy(); + std::pair Result = TLI.LowerCallTo(getRoot(), Type::VoidTy, false, false, false, CallingConv::C, true, DAG.getExternalSymbol("free", IntPtr), Args, DAG); @@ -4192,7 +4577,7 @@ void SelectionDAGLowering::visitVAStart(CallInst &I) { } void SelectionDAGLowering::visitVAArg(VAArgInst &I) { - SDOperand V = DAG.getVAArg(TLI.getValueType(I.getType()), getRoot(), + SDValue V = DAG.getVAArg(TLI.getValueType(I.getType()), getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0))); setValue(&I, V); @@ -4217,60 +4602,66 @@ void SelectionDAGLowering::visitVACopy(CallInst &I) { /// implementation, which just inserts a FORMAL_ARGUMENTS node. FIXME: When all /// targets are migrated to using FORMAL_ARGUMENTS, this hook should be /// integrated into SDISel. -std::vector -TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { +void TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG, + SmallVectorImpl &ArgValues) { // Add CC# and isVararg as operands to the FORMAL_ARGUMENTS node. - std::vector Ops; + SmallVector Ops; Ops.push_back(DAG.getRoot()); Ops.push_back(DAG.getConstant(F.getCallingConv(), getPointerTy())); Ops.push_back(DAG.getConstant(F.isVarArg(), getPointerTy())); // Add one result value for each formal argument. - std::vector RetVals; + SmallVector RetVals; unsigned j = 1; for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++j) { - MVT::ValueType VT = getValueType(I->getType()); - ISD::ArgFlagsTy Flags; - unsigned OriginalAlignment = - getTargetData()->getABITypeAlignment(I->getType()); - - if (F.paramHasAttr(j, ParamAttr::ZExt)) - Flags.setZExt(); - if (F.paramHasAttr(j, ParamAttr::SExt)) - Flags.setSExt(); - if (F.paramHasAttr(j, ParamAttr::InReg)) - Flags.setInReg(); - if (F.paramHasAttr(j, ParamAttr::StructRet)) - Flags.setSRet(); - if (F.paramHasAttr(j, ParamAttr::ByVal)) { - Flags.setByVal(); - const PointerType *Ty = cast(I->getType()); - const Type *ElementTy = Ty->getElementType(); - unsigned FrameAlign = getByValTypeAlignment(ElementTy); - unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); - // For ByVal, alignment should be passed from FE. BE will guess if - // this info is not there but there are cases it cannot get right. - if (F.getParamAlignment(j)) - FrameAlign = F.getParamAlignment(j); - Flags.setByValAlign(FrameAlign); - Flags.setByValSize(FrameSize); - } - if (F.paramHasAttr(j, ParamAttr::Nest)) - Flags.setNest(); - Flags.setOrigAlign(OriginalAlignment); + SmallVector ValueVTs; + ComputeValueVTs(*this, I->getType(), ValueVTs); + for (unsigned Value = 0, NumValues = ValueVTs.size(); + Value != NumValues; ++Value) { + MVT VT = ValueVTs[Value]; + const Type *ArgTy = VT.getTypeForMVT(); + ISD::ArgFlagsTy Flags; + unsigned OriginalAlignment = + getTargetData()->getABITypeAlignment(ArgTy); + + if (F.paramHasAttr(j, ParamAttr::ZExt)) + Flags.setZExt(); + if (F.paramHasAttr(j, ParamAttr::SExt)) + Flags.setSExt(); + if (F.paramHasAttr(j, ParamAttr::InReg)) + Flags.setInReg(); + if (F.paramHasAttr(j, ParamAttr::StructRet)) + Flags.setSRet(); + if (F.paramHasAttr(j, ParamAttr::ByVal)) { + Flags.setByVal(); + const PointerType *Ty = cast(I->getType()); + const Type *ElementTy = Ty->getElementType(); + unsigned FrameAlign = getByValTypeAlignment(ElementTy); + unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); + // For ByVal, alignment should be passed from FE. BE will guess if + // this info is not there but there are cases it cannot get right. + if (F.getParamAlignment(j)) + FrameAlign = F.getParamAlignment(j); + Flags.setByValAlign(FrameAlign); + Flags.setByValSize(FrameSize); + } + if (F.paramHasAttr(j, ParamAttr::Nest)) + Flags.setNest(); + Flags.setOrigAlign(OriginalAlignment); - MVT::ValueType RegisterVT = getRegisterType(VT); - unsigned NumRegs = getNumRegisters(VT); - for (unsigned i = 0; i != NumRegs; ++i) { - RetVals.push_back(RegisterVT); - ISD::ArgFlagsTy MyFlags = Flags; - if (NumRegs > 1 && i == 0) - MyFlags.setSplit(); - // if it isn't first piece, alignment must be 1 - else if (i > 0) - MyFlags.setOrigAlign(1); - Ops.push_back(DAG.getArgFlags(MyFlags)); + MVT RegisterVT = getRegisterType(VT); + unsigned NumRegs = getNumRegisters(VT); + for (unsigned i = 0; i != NumRegs; ++i) { + RetVals.push_back(RegisterVT); + ISD::ArgFlagsTy MyFlags = Flags; + if (NumRegs > 1 && i == 0) + MyFlags.setSplit(); + // if it isn't first piece, alignment must be 1 + else if (i > 0) + MyFlags.setOrigAlign(1); + Ops.push_back(DAG.getArgFlags(MyFlags)); + } } } @@ -4284,7 +4675,7 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { // Prelower FORMAL_ARGUMENTS. This isn't required for functionality, but // allows exposing the loads that may be part of the argument access to the // first DAGCombiner pass. - SDOperand TmpRes = LowerOperation(SDOperand(Result, 0), DAG); + SDValue TmpRes = LowerOperation(SDValue(Result, 0), DAG); // The number of results should match up, except that the lowered one may have // an extra flag result. @@ -4292,36 +4683,46 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { (Result->getNumValues()+1 == TmpRes.Val->getNumValues() && TmpRes.getValue(Result->getNumValues()).getValueType() == MVT::Flag)) && "Lowering produced unexpected number of results!"); + + // The FORMAL_ARGUMENTS node itself is likely no longer needed. + if (Result != TmpRes.Val && Result->use_empty()) { + HandleSDNode Dummy(DAG.getRoot()); + DAG.RemoveDeadNode(Result); + } + Result = TmpRes.Val; unsigned NumArgRegs = Result->getNumValues() - 1; - DAG.setRoot(SDOperand(Result, NumArgRegs)); + DAG.setRoot(SDValue(Result, NumArgRegs)); // Set up the return result vector. - Ops.clear(); unsigned i = 0; unsigned Idx = 1; for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++Idx) { - MVT::ValueType VT = getValueType(I->getType()); - MVT::ValueType PartVT = getRegisterType(VT); - - unsigned NumParts = getNumRegisters(VT); - SmallVector Parts(NumParts); - for (unsigned j = 0; j != NumParts; ++j) - Parts[j] = SDOperand(Result, i++); - - ISD::NodeType AssertOp = ISD::DELETED_NODE; - if (F.paramHasAttr(Idx, ParamAttr::SExt)) - AssertOp = ISD::AssertSext; - else if (F.paramHasAttr(Idx, ParamAttr::ZExt)) - AssertOp = ISD::AssertZext; - - Ops.push_back(getCopyFromParts(DAG, &Parts[0], NumParts, PartVT, VT, - AssertOp)); + SmallVector ValueVTs; + ComputeValueVTs(*this, I->getType(), ValueVTs); + for (unsigned Value = 0, NumValues = ValueVTs.size(); + Value != NumValues; ++Value) { + MVT VT = ValueVTs[Value]; + MVT PartVT = getRegisterType(VT); + + unsigned NumParts = getNumRegisters(VT); + SmallVector Parts(NumParts); + for (unsigned j = 0; j != NumParts; ++j) + Parts[j] = SDValue(Result, i++); + + ISD::NodeType AssertOp = ISD::DELETED_NODE; + if (F.paramHasAttr(Idx, ParamAttr::SExt)) + AssertOp = ISD::AssertSext; + else if (F.paramHasAttr(Idx, ParamAttr::ZExt)) + AssertOp = ISD::AssertZext; + + ArgValues.push_back(getCopyFromParts(DAG, &Parts[0], NumParts, PartVT, VT, + AssertOp)); + } } assert(i == NumArgRegs && "Argument register count mismatch!"); - return Ops; } @@ -4329,13 +4730,13 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { /// implementation, which just inserts an ISD::CALL node, which is later custom /// lowered by the target to something concrete. FIXME: When all targets are /// migrated to using ISD::CALL, this hook should be integrated into SDISel. -std::pair -TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, +std::pair +TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, bool RetSExt, bool RetZExt, bool isVarArg, unsigned CallingConv, bool isTailCall, - SDOperand Callee, + SDValue Callee, ArgListTy &Args, SelectionDAG &DAG) { - SmallVector Ops; + SmallVector Ops; Ops.push_back(Chain); // Op#0 - Chain Ops.push_back(DAG.getConstant(CallingConv, getPointerTy())); // Op#1 - CC Ops.push_back(DAG.getConstant(isVarArg, getPointerTy())); // Op#2 - VarArg @@ -4344,72 +4745,78 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, // Handle all of the outgoing arguments. for (unsigned i = 0, e = Args.size(); i != e; ++i) { - MVT::ValueType VT = getValueType(Args[i].Ty); - SDOperand Op = Args[i].Node; - ISD::ArgFlagsTy Flags; - unsigned OriginalAlignment = - getTargetData()->getABITypeAlignment(Args[i].Ty); - - if (Args[i].isZExt) - Flags.setZExt(); - if (Args[i].isSExt) - Flags.setSExt(); - if (Args[i].isInReg) - Flags.setInReg(); - if (Args[i].isSRet) - Flags.setSRet(); - if (Args[i].isByVal) { - Flags.setByVal(); - const PointerType *Ty = cast(Args[i].Ty); - const Type *ElementTy = Ty->getElementType(); - unsigned FrameAlign = getByValTypeAlignment(ElementTy); - unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); - // For ByVal, alignment should come from FE. BE will guess if this - // info is not there but there are cases it cannot get right. - if (Args[i].Alignment) - FrameAlign = Args[i].Alignment; - Flags.setByValAlign(FrameAlign); - Flags.setByValSize(FrameSize); - } - if (Args[i].isNest) - Flags.setNest(); - Flags.setOrigAlign(OriginalAlignment); - - MVT::ValueType PartVT = getRegisterType(VT); - unsigned NumParts = getNumRegisters(VT); - SmallVector Parts(NumParts); - ISD::NodeType ExtendKind = ISD::ANY_EXTEND; - - if (Args[i].isSExt) - ExtendKind = ISD::SIGN_EXTEND; - else if (Args[i].isZExt) - ExtendKind = ISD::ZERO_EXTEND; - - getCopyToParts(DAG, Op, &Parts[0], NumParts, PartVT, ExtendKind); - - for (unsigned i = 0; i != NumParts; ++i) { - // if it isn't first piece, alignment must be 1 - ISD::ArgFlagsTy MyFlags = Flags; - if (NumParts > 1 && i == 0) - MyFlags.setSplit(); - else if (i != 0) - MyFlags.setOrigAlign(1); - - Ops.push_back(Parts[i]); - Ops.push_back(DAG.getArgFlags(MyFlags)); + SmallVector ValueVTs; + ComputeValueVTs(*this, Args[i].Ty, ValueVTs); + for (unsigned Value = 0, NumValues = ValueVTs.size(); + Value != NumValues; ++Value) { + MVT VT = ValueVTs[Value]; + const Type *ArgTy = VT.getTypeForMVT(); + SDValue Op = SDValue(Args[i].Node.Val, Args[i].Node.ResNo + Value); + ISD::ArgFlagsTy Flags; + unsigned OriginalAlignment = + getTargetData()->getABITypeAlignment(ArgTy); + + if (Args[i].isZExt) + Flags.setZExt(); + if (Args[i].isSExt) + Flags.setSExt(); + if (Args[i].isInReg) + Flags.setInReg(); + if (Args[i].isSRet) + Flags.setSRet(); + if (Args[i].isByVal) { + Flags.setByVal(); + const PointerType *Ty = cast(Args[i].Ty); + const Type *ElementTy = Ty->getElementType(); + unsigned FrameAlign = getByValTypeAlignment(ElementTy); + unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); + // For ByVal, alignment should come from FE. BE will guess if this + // info is not there but there are cases it cannot get right. + if (Args[i].Alignment) + FrameAlign = Args[i].Alignment; + Flags.setByValAlign(FrameAlign); + Flags.setByValSize(FrameSize); + } + if (Args[i].isNest) + Flags.setNest(); + Flags.setOrigAlign(OriginalAlignment); + + MVT PartVT = getRegisterType(VT); + unsigned NumParts = getNumRegisters(VT); + SmallVector Parts(NumParts); + ISD::NodeType ExtendKind = ISD::ANY_EXTEND; + + if (Args[i].isSExt) + ExtendKind = ISD::SIGN_EXTEND; + else if (Args[i].isZExt) + ExtendKind = ISD::ZERO_EXTEND; + + getCopyToParts(DAG, Op, &Parts[0], NumParts, PartVT, ExtendKind); + + for (unsigned i = 0; i != NumParts; ++i) { + // if it isn't first piece, alignment must be 1 + ISD::ArgFlagsTy MyFlags = Flags; + if (NumParts > 1 && i == 0) + MyFlags.setSplit(); + else if (i != 0) + MyFlags.setOrigAlign(1); + + Ops.push_back(Parts[i]); + Ops.push_back(DAG.getArgFlags(MyFlags)); + } } } // Figure out the result value types. We start by making a list of // the potentially illegal return value types. - SmallVector LoweredRetTys; - SmallVector RetTys; + SmallVector LoweredRetTys; + SmallVector RetTys; ComputeValueVTs(*this, RetTy, RetTys); // Then we translate that to a list of legal types. for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { - MVT::ValueType VT = RetTys[I]; - MVT::ValueType RegisterVT = getRegisterType(VT); + MVT VT = RetTys[I]; + MVT RegisterVT = getRegisterType(VT); unsigned NumRegs = getNumRegisters(VT); for (unsigned i = 0; i != NumRegs; ++i) LoweredRetTys.push_back(RegisterVT); @@ -4418,7 +4825,7 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, LoweredRetTys.push_back(MVT::Other); // Always has a chain. // Create the CALL node. - SDOperand Res = DAG.getNode(ISD::CALL, + SDValue Res = DAG.getNode(ISD::CALL, DAG.getVTList(&LoweredRetTys[0], LoweredRetTys.size()), &Ops[0], Ops.size()); @@ -4433,48 +4840,40 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, else if (RetZExt) AssertOp = ISD::AssertZext; - SmallVector ReturnValues; + SmallVector ReturnValues; unsigned RegNo = 0; for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { - MVT::ValueType VT = RetTys[I]; - MVT::ValueType RegisterVT = getRegisterType(VT); + MVT VT = RetTys[I]; + MVT RegisterVT = getRegisterType(VT); unsigned NumRegs = getNumRegisters(VT); unsigned RegNoEnd = NumRegs + RegNo; - SmallVector Results; + SmallVector Results; for (; RegNo != RegNoEnd; ++RegNo) Results.push_back(Res.getValue(RegNo)); - SDOperand ReturnValue = + SDValue ReturnValue = getCopyFromParts(DAG, &Results[0], NumRegs, RegisterVT, VT, AssertOp); ReturnValues.push_back(ReturnValue); } - Res = ReturnValues.size() == 1 ? ReturnValues.front() : - DAG.getNode(ISD::MERGE_VALUES, - DAG.getVTList(&RetTys[0], RetTys.size()), - &ReturnValues[0], ReturnValues.size()); + Res = DAG.getMergeValues(DAG.getVTList(&RetTys[0], RetTys.size()), + &ReturnValues[0], ReturnValues.size()); } return std::make_pair(Res, Chain); } -SDOperand TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { +SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) { assert(0 && "LowerOperation not implemented for this target!"); abort(); - return SDOperand(); + return SDValue(); } -SDOperand TargetLowering::CustomPromoteOperation(SDOperand Op, - SelectionDAG &DAG) { - assert(0 && "CustomPromoteOperation not implemented for this target!"); - abort(); - return SDOperand(); -} //===----------------------------------------------------------------------===// // SelectionDAGISel code //===----------------------------------------------------------------------===// -unsigned SelectionDAGISel::MakeReg(MVT::ValueType VT) { +unsigned SelectionDAGISel::MakeReg(MVT VT) { return RegInfo->createVirtualRegister(TLI.getRegClassFor(VT)); } @@ -4484,8 +4883,6 @@ void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); } - - bool SelectionDAGISel::runOnFunction(Function &Fn) { // Get alias analysis for load/store combining. AA = &getAnalysis(); @@ -4505,8 +4902,7 @@ bool SelectionDAGISel::runOnFunction(Function &Fn) { // Mark landing pad. FuncInfo.MBBMap[Invoke->getSuccessor(1)]->setIsLandingPad(); - for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) - SelectBasicBlock(I, MF, FuncInfo); + SelectAllBasicBlocks(Fn, MF, FuncInfo); // Add function live-ins to entry block live-in set. BasicBlock *EntryBB = &Fn.getEntryBlock(); @@ -4524,16 +4920,15 @@ bool SelectionDAGISel::runOnFunction(Function &Fn) { return true; } -void SelectionDAGLowering::CopyValueToVirtualRegister(Value *V, - unsigned Reg) { - SDOperand Op = getValue(V); +void SelectionDAGLowering::CopyValueToVirtualRegister(Value *V, unsigned Reg) { + SDValue Op = getValue(V); assert((Op.getOpcode() != ISD::CopyFromReg || cast(Op.getOperand(1))->getReg() != Reg) && "Copy from a reg to the same reg!"); assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg"); RegsForValue RFV(TLI, Reg, V->getType()); - SDOperand Chain = DAG.getEntryNode(); + SDValue Chain = DAG.getEntryNode(); RFV.getCopyToRegs(Op, DAG, Chain, 0); PendingExports.push_back(Chain); } @@ -4543,15 +4938,18 @@ LowerArguments(BasicBlock *LLVMBB, SelectionDAGLowering &SDL) { // If this is the entry block, emit arguments. Function &F = *LLVMBB->getParent(); FunctionLoweringInfo &FuncInfo = SDL.FuncInfo; - SDOperand OldRoot = SDL.DAG.getRoot(); - std::vector Args = TLI.LowerArguments(F, SDL.DAG); + SDValue OldRoot = SDL.DAG.getRoot(); + SmallVector Args; + TLI.LowerArguments(F, SDL.DAG, Args); unsigned a = 0; for (Function::arg_iterator AI = F.arg_begin(), E = F.arg_end(); - AI != E; ++AI, ++a) + AI != E; ++AI) { + SmallVector ValueVTs; + ComputeValueVTs(TLI, AI->getType(), ValueVTs); + unsigned NumValues = ValueVTs.size(); if (!AI->use_empty()) { - SDL.setValue(AI, Args[a]); - + SDL.setValue(AI, SDL.DAG.getMergeValues(&Args[a], NumValues)); // If this argument is live outside of the entry block, insert a copy from // whereever we got it to the vreg that other BB's will reference it as. DenseMap::iterator VMI=FuncInfo.ValueMap.find(AI); @@ -4559,6 +4957,8 @@ LowerArguments(BasicBlock *LLVMBB, SelectionDAGLowering &SDL) { SDL.CopyValueToVirtualRegister(AI, VMI->second); } } + a += NumValues; + } // Finally, if the target has anything special to do, allow it to do so. // FIXME: this should insert code into the DAG! @@ -4578,12 +4978,46 @@ static void copyCatchInfo(BasicBlock *SrcBB, BasicBlock *DestBB, } } +/// IsFixedFrameObjectWithPosOffset - Check if object is a fixed frame object and +/// whether object offset >= 0. +static bool +IsFixedFrameObjectWithPosOffset(MachineFrameInfo * MFI, SDValue Op) { + if (!isa(Op)) return false; + + FrameIndexSDNode * FrameIdxNode = dyn_cast(Op); + int FrameIdx = FrameIdxNode->getIndex(); + return MFI->isFixedObjectIndex(FrameIdx) && + MFI->getObjectOffset(FrameIdx) >= 0; +} + +/// IsPossiblyOverwrittenArgumentOfTailCall - Check if the operand could +/// possibly be overwritten when lowering the outgoing arguments in a tail +/// call. Currently the implementation of this call is very conservative and +/// assumes all arguments sourcing from FORMAL_ARGUMENTS or a CopyFromReg with +/// virtual registers would be overwritten by direct lowering. +static bool IsPossiblyOverwrittenArgumentOfTailCall(SDValue Op, + MachineFrameInfo * MFI) { + RegisterSDNode * OpReg = NULL; + if (Op.getOpcode() == ISD::FORMAL_ARGUMENTS || + (Op.getOpcode()== ISD::CopyFromReg && + (OpReg = dyn_cast(Op.getOperand(1))) && + (OpReg->getReg() >= TargetRegisterInfo::FirstVirtualRegister)) || + (Op.getOpcode() == ISD::LOAD && + IsFixedFrameObjectWithPosOffset(MFI, Op.getOperand(1))) || + (Op.getOpcode() == ISD::MERGE_VALUES && + Op.getOperand(Op.ResNo).getOpcode() == ISD::LOAD && + IsFixedFrameObjectWithPosOffset(MFI, Op.getOperand(Op.ResNo). + getOperand(1)))) + return true; + return false; +} + /// CheckDAGForTailCallsAndFixThem - This Function looks for CALL nodes in the /// DAG and fixes their tailcall attribute operand. static void CheckDAGForTailCallsAndFixThem(SelectionDAG &DAG, TargetLowering& TLI) { SDNode * Ret = NULL; - SDOperand Terminator = DAG.getRoot(); + SDValue Terminator = DAG.getRoot(); // Find RET node. if (Terminator.getOpcode() == ISD::RET) { @@ -4592,29 +5026,62 @@ static void CheckDAGForTailCallsAndFixThem(SelectionDAG &DAG, // Fix tail call attribute of CALL nodes. for (SelectionDAG::allnodes_iterator BE = DAG.allnodes_begin(), - BI = prior(DAG.allnodes_end()); BI != BE; --BI) { + BI = DAG.allnodes_end(); BI != BE; ) { + --BI; if (BI->getOpcode() == ISD::CALL) { - SDOperand OpRet(Ret, 0); - SDOperand OpCall(static_cast(BI), 0); + SDValue OpRet(Ret, 0); + SDValue OpCall(BI, 0); bool isMarkedTailCall = cast(OpCall.getOperand(3))->getValue() != 0; // If CALL node has tail call attribute set to true and the call is not // eligible (no RET or the target rejects) the attribute is fixed to // false. The TargetLowering::IsEligibleForTailCallOptimization function // must correctly identify tail call optimizable calls. - if (isMarkedTailCall && - (Ret==NULL || - !TLI.IsEligibleForTailCallOptimization(OpCall, OpRet, DAG))) { - SmallVector Ops; + if (!isMarkedTailCall) continue; + if (Ret==NULL || + !TLI.IsEligibleForTailCallOptimization(OpCall, OpRet, DAG)) { + // Not eligible. Mark CALL node as non tail call. + SmallVector Ops; unsigned idx=0; - for(SDNode::op_iterator I =OpCall.Val->op_begin(), - E=OpCall.Val->op_end(); I!=E; I++, idx++) { + for(SDNode::op_iterator I =OpCall.Val->op_begin(), + E = OpCall.Val->op_end(); I != E; I++, idx++) { if (idx!=3) Ops.push_back(*I); - else + else Ops.push_back(DAG.getConstant(false, TLI.getPointerTy())); } DAG.UpdateNodeOperands(OpCall, Ops.begin(), Ops.size()); + } else { + // Look for tail call clobbered arguments. Emit a series of + // copyto/copyfrom virtual register nodes to protect them. + SmallVector Ops; + SDValue Chain = OpCall.getOperand(0), InFlag; + unsigned idx=0; + for(SDNode::op_iterator I = OpCall.Val->op_begin(), + E = OpCall.Val->op_end(); I != E; I++, idx++) { + SDValue Arg = *I; + if (idx > 4 && (idx % 2)) { + bool isByVal = cast(OpCall.getOperand(idx+1))-> + getArgFlags().isByVal(); + MachineFunction &MF = DAG.getMachineFunction(); + MachineFrameInfo *MFI = MF.getFrameInfo(); + if (!isByVal && + IsPossiblyOverwrittenArgumentOfTailCall(Arg, MFI)) { + MVT VT = Arg.getValueType(); + unsigned VReg = MF.getRegInfo(). + createVirtualRegister(TLI.getRegClassFor(VT)); + Chain = DAG.getCopyToReg(Chain, VReg, Arg, InFlag); + InFlag = Chain.getValue(1); + Arg = DAG.getCopyFromReg(Chain, VReg, VT, InFlag); + Chain = Arg.getValue(1); + InFlag = Arg.getValue(2); + } + } + Ops.push_back(Arg); + } + // Link in chain of CopyTo/CopyFromReg. + Ops[0] = Chain; + DAG.UpdateNodeOperands(OpCall, Ops.begin(), Ops.size()); } } } @@ -4638,9 +5105,7 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, // Add a label to mark the beginning of the landing pad. Deletion of the // landing pad can thus be detected via the MachineModuleInfo. unsigned LabelID = MMI->addLandingPad(BB); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, DAG.getEntryNode(), - DAG.getConstant(LabelID, MVT::i32), - DAG.getConstant(1, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::EH_LABEL, DAG.getEntryNode(), LabelID)); // Mark exception register as live in. unsigned Reg = TLI.getExceptionAddressRegister(); @@ -4753,10 +5218,15 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, // Remember that this register needs to added to the machine PHI node as // the input for this MBB. - MVT::ValueType VT = TLI.getValueType(PN->getType()); - unsigned NumRegisters = TLI.getNumRegisters(VT); - for (unsigned i = 0, e = NumRegisters; i != e; ++i) - PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); + SmallVector ValueVTs; + ComputeValueVTs(TLI, PN->getType(), ValueVTs); + for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) { + MVT VT = ValueVTs[vti]; + unsigned NumRegisters = TLI.getNumRegisters(VT); + for (unsigned i = 0, e = NumRegisters; i != e; ++i) + PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); + Reg += NumRegisters; + } } } ConstantsOut.clear(); @@ -4783,48 +5253,209 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, CheckDAGForTailCallsAndFixThem(DAG, TLI); } +void SelectionDAGISel::ComputeLiveOutVRegInfo(SelectionDAG &DAG) { + SmallPtrSet VisitedNodes; + SmallVector Worklist; + + Worklist.push_back(DAG.getRoot().Val); + + APInt Mask; + APInt KnownZero; + APInt KnownOne; + + while (!Worklist.empty()) { + SDNode *N = Worklist.back(); + Worklist.pop_back(); + + // If we've already seen this node, ignore it. + if (!VisitedNodes.insert(N)) + continue; + + // Otherwise, add all chain operands to the worklist. + for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) + if (N->getOperand(i).getValueType() == MVT::Other) + Worklist.push_back(N->getOperand(i).Val); + + // If this is a CopyToReg with a vreg dest, process it. + if (N->getOpcode() != ISD::CopyToReg) + continue; + + unsigned DestReg = cast(N->getOperand(1))->getReg(); + if (!TargetRegisterInfo::isVirtualRegister(DestReg)) + continue; + + // Ignore non-scalar or non-integer values. + SDValue Src = N->getOperand(2); + MVT SrcVT = Src.getValueType(); + if (!SrcVT.isInteger() || SrcVT.isVector()) + continue; + + unsigned NumSignBits = DAG.ComputeNumSignBits(Src); + Mask = APInt::getAllOnesValue(SrcVT.getSizeInBits()); + DAG.ComputeMaskedBits(Src, Mask, KnownZero, KnownOne); + + // Only install this information if it tells us something. + if (NumSignBits != 1 || KnownZero != 0 || KnownOne != 0) { + DestReg -= TargetRegisterInfo::FirstVirtualRegister; + FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); + if (DestReg >= FLI.LiveOutRegInfo.size()) + FLI.LiveOutRegInfo.resize(DestReg+1); + FunctionLoweringInfo::LiveOutInfo &LOI = FLI.LiveOutRegInfo[DestReg]; + LOI.NumSignBits = NumSignBits; + LOI.KnownOne = NumSignBits; + LOI.KnownZero = NumSignBits; + } + } +} + void SelectionDAGISel::CodeGenAndEmitDAG(SelectionDAG &DAG) { - DOUT << "Lowered selection DAG:\n"; + std::string GroupName; + if (TimePassesIsEnabled) + GroupName = "Instruction Selection and Scheduling"; + std::string BlockName; + if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs || + ViewDAGCombine2 || ViewISelDAGs || ViewSchedDAGs || ViewSUnitDAGs) + BlockName = DAG.getMachineFunction().getFunction()->getName() + ':' + + BB->getBasicBlock()->getName(); + + DOUT << "Initial selection DAG:\n"; DEBUG(DAG.dump()); + if (ViewDAGCombine1) DAG.viewGraph("dag-combine1 input for " + BlockName); + // Run the DAG combiner in pre-legalize mode. - DAG.Combine(false, *AA); + if (TimePassesIsEnabled) { + NamedRegionTimer T("DAG Combining 1", GroupName); + DAG.Combine(false, *AA); + } else { + DAG.Combine(false, *AA); + } DOUT << "Optimized lowered selection DAG:\n"; DEBUG(DAG.dump()); // Second step, hack on the DAG until it only uses operations and types that // the target supports. -#if 0 // Enable this some day. - DAG.LegalizeTypes(); - // Someday even later, enable a dag combine pass here. -#endif - DAG.Legalize(); + if (EnableLegalizeTypes) {// Enable this some day. + if (ViewLegalizeTypesDAGs) DAG.viewGraph("legalize-types input for " + + BlockName); + + if (TimePassesIsEnabled) { + NamedRegionTimer T("Type Legalization", GroupName); + DAG.LegalizeTypes(); + } else { + DAG.LegalizeTypes(); + } + + DOUT << "Type-legalized selection DAG:\n"; + DEBUG(DAG.dump()); + + // TODO: enable a dag combine pass here. + } + + if (ViewLegalizeDAGs) DAG.viewGraph("legalize input for " + BlockName); + + if (TimePassesIsEnabled) { + NamedRegionTimer T("DAG Legalization", GroupName); + DAG.Legalize(); + } else { + DAG.Legalize(); + } DOUT << "Legalized selection DAG:\n"; DEBUG(DAG.dump()); + if (ViewDAGCombine2) DAG.viewGraph("dag-combine2 input for " + BlockName); + // Run the DAG combiner in post-legalize mode. - DAG.Combine(true, *AA); + if (TimePassesIsEnabled) { + NamedRegionTimer T("DAG Combining 2", GroupName); + DAG.Combine(true, *AA); + } else { + DAG.Combine(true, *AA); + } DOUT << "Optimized legalized selection DAG:\n"; DEBUG(DAG.dump()); - if (ViewISelDAGs) DAG.viewGraph(); + if (ViewISelDAGs) DAG.viewGraph("isel input for " + BlockName); + + if (!FastISel && EnableValueProp) + ComputeLiveOutVRegInfo(DAG); // Third, instruction select all of the operations to machine code, adding the // code to the MachineBasicBlock. - InstructionSelectBasicBlock(DAG); + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Selection", GroupName); + InstructionSelect(DAG); + } else { + InstructionSelect(DAG); + } + + DOUT << "Selected selection DAG:\n"; + DEBUG(DAG.dump()); + + if (ViewSchedDAGs) DAG.viewGraph("scheduler input for " + BlockName); + + // Schedule machine code. + ScheduleDAG *Scheduler; + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Scheduling", GroupName); + Scheduler = Schedule(DAG); + } else { + Scheduler = Schedule(DAG); + } + + if (ViewSUnitDAGs) Scheduler->viewGraph(); + + // Emit machine code to BB. This can change 'BB' to the last block being + // inserted into. + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Creation", GroupName); + BB = Scheduler->EmitSchedule(); + } else { + BB = Scheduler->EmitSchedule(); + } + + // Free the scheduler state. + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Scheduling Cleanup", GroupName); + delete Scheduler; + } else { + delete Scheduler; + } + + // Perform target specific isel post processing. + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Selection Post Processing", GroupName); + InstructionSelectPostProcessing(); + } else { + InstructionSelectPostProcessing(); + } DOUT << "Selected machine code:\n"; DEBUG(BB->dump()); } -void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, - FunctionLoweringInfo &FuncInfo) { +void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, MachineFunction &MF, + FunctionLoweringInfo &FuncInfo) { + // Define NodeAllocator here so that memory allocation is reused for + // each basic block. + NodeAllocatorType NodeAllocator; + + for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) + SelectBasicBlock(I, MF, FuncInfo, NodeAllocator); +} + +void +SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, + FunctionLoweringInfo &FuncInfo, + NodeAllocatorType &NodeAllocator) { std::vector > PHINodesToUpdate; { - SelectionDAG DAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG DAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &DAG; // First step, lower LLVM code to some DAG. This DAG may use operations and @@ -4858,7 +5489,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i) { // Lower header first, if it wasn't already lowered if (!BitTestCases[i].Emitted) { - SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG HSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &HSDAG; SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -4871,7 +5504,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, } for (unsigned j = 0, ej = BitTestCases[i].Cases.size(); j != ej; ++j) { - SelectionDAG BSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG BSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &BSDAG; SelectionDAGLowering BSDL(BSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -4928,7 +5563,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, for (unsigned i = 0, e = JTCases.size(); i != e; ++i) { // Lower header first, if it wasn't already lowered if (!JTCases[i].first.Emitted) { - SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG HSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &HSDAG; SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -4940,7 +5577,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, CodeGenAndEmitDAG(HSDAG); } - SelectionDAG JSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG JSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &JSDAG; SelectionDAGLowering JSDL(JSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -4988,7 +5627,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, // If we generated any switch lowering information, build and codegen any // additional DAGs necessary. for (unsigned i = 0, e = SwitchCases.size(); i != e; ++i) { - SelectionDAG SDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG SDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &SDAG; SelectionDAGLowering SDL(SDAG, TLI, *AA, FuncInfo, GCI); @@ -5033,12 +5674,10 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, } -//===----------------------------------------------------------------------===// -/// ScheduleAndEmitDAG - Pick a safe ordering and emit instructions for each +/// Schedule - Pick a safe ordering for instructions for each /// target node in the graph. -void SelectionDAGISel::ScheduleAndEmitDAG(SelectionDAG &DAG) { - if (ViewSchedDAGs) DAG.viewGraph(); - +/// +ScheduleDAG *SelectionDAGISel::Schedule(SelectionDAG &DAG) { RegisterScheduler::FunctionPassCtor Ctor = RegisterScheduler::getDefault(); if (!Ctor) { @@ -5046,12 +5685,10 @@ void SelectionDAGISel::ScheduleAndEmitDAG(SelectionDAG &DAG) { RegisterScheduler::setDefault(Ctor); } - ScheduleDAG *SL = Ctor(this, &DAG, BB); - BB = SL->Run(); - - if (ViewSUnitDAGs) SL->viewGraph(); + ScheduleDAG *Scheduler = Ctor(this, &DAG, BB, FastISel); + Scheduler->Run(); - delete SL; + return Scheduler; } @@ -5068,7 +5705,7 @@ HazardRecognizer *SelectionDAGISel::CreateTargetHazardRecognizer() { /// the dag combiner simplified the 255, we still want to match. RHS is the /// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value /// specified in the .td file (e.g. 255). -bool SelectionDAGISel::CheckAndMask(SDOperand LHS, ConstantSDNode *RHS, +bool SelectionDAGISel::CheckAndMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const { const APInt &ActualMask = RHS->getAPIntValue(); const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS); @@ -5097,7 +5734,7 @@ bool SelectionDAGISel::CheckAndMask(SDOperand LHS, ConstantSDNode *RHS, /// the dag combiner simplified the 255, we still want to match. RHS is the /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value /// specified in the .td file (e.g. 255). -bool SelectionDAGISel::CheckOrMask(SDOperand LHS, ConstantSDNode *RHS, +bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const { const APInt &ActualMask = RHS->getAPIntValue(); const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS); @@ -5131,8 +5768,8 @@ bool SelectionDAGISel::CheckOrMask(SDOperand LHS, ConstantSDNode *RHS, /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated /// by tblgen. Others should not call it. void SelectionDAGISel:: -SelectInlineAsmMemoryOperands(std::vector &Ops, SelectionDAG &DAG) { - std::vector InOps; +SelectInlineAsmMemoryOperands(std::vector &Ops, SelectionDAG &DAG) { + std::vector InOps; std::swap(InOps, Ops); Ops.push_back(InOps[0]); // input chain. @@ -5151,14 +5788,14 @@ SelectInlineAsmMemoryOperands(std::vector &Ops, SelectionDAG &DAG) { } else { assert((Flags >> 3) == 1 && "Memory operand with multiple values?"); // Otherwise, this is a memory operand. Ask the target to select it. - std::vector SelOps; + std::vector SelOps; if (SelectInlineAsmMemoryOperand(InOps[i+1], 'm', SelOps, DAG)) { cerr << "Could not match memory address. Inline asm failure!\n"; exit(1); } // Add this to the output node. - MVT::ValueType IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); + MVT IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); Ops.push_back(DAG.getTargetConstant(4/*MEM*/ | (SelOps.size() << 3), IntPtrTy)); Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());