X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAGISel.cpp;h=6299bc4b851f7270dc982993b8f561ab926a438a;hb=bd3ba461eb5578a81ba09ff7bd7eb271d1130196;hp=94386c62851f485f9a4bafc5853147e1b6b40b9f;hpb=4468c1fd1c3dcc2cca7b881eb9f37f018405d2d9;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 94386c62851..6299bc4b851 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -42,14 +42,35 @@ #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" -#include "llvm/Support/MathExtras.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/Timer.h" #include using namespace llvm; +static cl::opt +EnableValueProp("enable-value-prop", cl::Hidden); +static cl::opt +EnableLegalizeTypes("enable-legalize-types", cl::Hidden); + + #ifndef NDEBUG static cl::opt +ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, + cl::desc("Pop up a window to show dags before the first " + "dag combine pass")); +static cl::opt +ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, + cl::desc("Pop up a window to show dags before legalize types")); +static cl::opt +ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, + cl::desc("Pop up a window to show dags before legalize")); +static cl::opt +ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, + cl::desc("Pop up a window to show dags before the second " + "dag combine pass")); +static cl::opt ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected")); static cl::opt @@ -59,7 +80,11 @@ static cl::opt ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed")); #else -static const bool ViewISelDAGs = 0, ViewSchedDAGs = 0, ViewSUnitDAGs = 0; +static const bool ViewDAGCombine1 = false, + ViewLegalizeTypesDAGs = false, ViewLegalizeDAGs = false, + ViewDAGCombine2 = false, + ViewISelDAGs = false, ViewSchedDAGs = false, + ViewSUnitDAGs = false; #endif //===---------------------------------------------------------------------===// @@ -74,70 +99,183 @@ MachinePassRegistry RegisterScheduler::Registry; /// ISHeuristic command line option for instruction schedulers. /// //===---------------------------------------------------------------------===// -namespace { - cl::opt > - ISHeuristic("pre-RA-sched", - cl::init(&createDefaultScheduler), - cl::desc("Instruction schedulers available (before register" - " allocation):")); - - static RegisterScheduler - defaultListDAGScheduler("default", " Best scheduler for the target", - createDefaultScheduler); -} // namespace +static cl::opt > +ISHeuristic("pre-RA-sched", + cl::init(&createDefaultScheduler), + cl::desc("Instruction schedulers available (before register" + " allocation):")); + +static RegisterScheduler +defaultListDAGScheduler("default", " Best scheduler for the target", + createDefaultScheduler); namespace { struct SDISelAsmOperandInfo; } +/// ComputeLinearIndex - Given an LLVM IR aggregate type and a sequence +/// insertvalue or extractvalue indices that identify a member, return +/// the linearized index of the start of the member. +/// +static unsigned ComputeLinearIndex(const TargetLowering &TLI, const Type *Ty, + const unsigned *Indices, + const unsigned *IndicesEnd, + unsigned CurIndex = 0) { + // Base case: We're done. + if (Indices && Indices == IndicesEnd) + return CurIndex; + + // Given a struct type, recursively traverse the elements. + if (const StructType *STy = dyn_cast(Ty)) { + for (StructType::element_iterator EB = STy->element_begin(), + EI = EB, + EE = STy->element_end(); + EI != EE; ++EI) { + if (Indices && *Indices == unsigned(EI - EB)) + return ComputeLinearIndex(TLI, *EI, Indices+1, IndicesEnd, CurIndex); + CurIndex = ComputeLinearIndex(TLI, *EI, 0, 0, CurIndex); + } + } + // Given an array type, recursively traverse the elements. + else if (const ArrayType *ATy = dyn_cast(Ty)) { + const Type *EltTy = ATy->getElementType(); + for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) { + if (Indices && *Indices == i) + return ComputeLinearIndex(TLI, EltTy, Indices+1, IndicesEnd, CurIndex); + CurIndex = ComputeLinearIndex(TLI, EltTy, 0, 0, CurIndex); + } + } + // We haven't found the type we're looking for, so keep searching. + return CurIndex + 1; +} + +/// ComputeValueVTs - Given an LLVM IR type, compute a sequence of +/// MVTs that represent all the individual underlying +/// non-aggregate types that comprise it. +/// +/// If Offsets is non-null, it points to a vector to be filled in +/// with the in-memory offsets of each of the individual values. +/// +static void ComputeValueVTs(const TargetLowering &TLI, const Type *Ty, + SmallVectorImpl &ValueVTs, + SmallVectorImpl *Offsets = 0, + uint64_t StartingOffset = 0) { + // Given a struct type, recursively traverse the elements. + if (const StructType *STy = dyn_cast(Ty)) { + const StructLayout *SL = TLI.getTargetData()->getStructLayout(STy); + for (StructType::element_iterator EB = STy->element_begin(), + EI = EB, + EE = STy->element_end(); + EI != EE; ++EI) + ComputeValueVTs(TLI, *EI, ValueVTs, Offsets, + StartingOffset + SL->getElementOffset(EI - EB)); + return; + } + // Given an array type, recursively traverse the elements. + if (const ArrayType *ATy = dyn_cast(Ty)) { + const Type *EltTy = ATy->getElementType(); + uint64_t EltSize = TLI.getTargetData()->getABITypeSize(EltTy); + for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) + ComputeValueVTs(TLI, EltTy, ValueVTs, Offsets, + StartingOffset + i * EltSize); + return; + } + // Base case: we can get an MVT for this LLVM IR type. + ValueVTs.push_back(TLI.getValueType(Ty)); + if (Offsets) + Offsets->push_back(StartingOffset); +} + namespace { - /// RegsForValue - This struct represents the physical registers that a - /// particular value is assigned and the type information about the value. - /// This is needed because values can be promoted into larger registers and - /// expanded into multiple smaller registers than the value. + /// RegsForValue - This struct represents the registers (physical or virtual) + /// that a particular set of values is assigned, and the type information about + /// the value. The most common situation is to represent one value at a time, + /// but struct or array values are handled element-wise as multiple values. + /// The splitting of aggregates is performed recursively, so that we never + /// have aggregate-typed registers. The values at this point do not necessarily + /// have legal types, so each value may require one or more registers of some + /// legal type. + /// struct VISIBILITY_HIDDEN RegsForValue { - /// Regs - This list holds the register (for legal and promoted values) - /// or register set (for expanded values) that the value should be assigned - /// to. - std::vector Regs; + /// TLI - The TargetLowering object. + /// + const TargetLowering *TLI; + + /// ValueVTs - The value types of the values, which may not be legal, and + /// may need be promoted or synthesized from one or more registers. + /// + SmallVector ValueVTs; - /// RegVT - The value type of each register. + /// RegVTs - The value types of the registers. This is the same size as + /// ValueVTs and it records, for each value, what the type of the assigned + /// register or registers are. (Individual values are never synthesized + /// from more than one type of register.) /// - MVT::ValueType RegVT; + /// With virtual registers, the contents of RegVTs is redundant with TLI's + /// getRegisterType member function, however when with physical registers + /// it is necessary to have a separate record of the types. + /// + SmallVector RegVTs; - /// ValueVT - The value type of the LLVM value, which may be promoted from - /// RegVT or made from merging the two expanded parts. - MVT::ValueType ValueVT; + /// Regs - This list holds the registers assigned to the values. + /// Each legal or promoted value requires one register, and each + /// expanded value requires multiple registers. + /// + SmallVector Regs; - RegsForValue() : RegVT(MVT::Other), ValueVT(MVT::Other) {} + RegsForValue() : TLI(0) {} - RegsForValue(unsigned Reg, MVT::ValueType regvt, MVT::ValueType valuevt) - : RegVT(regvt), ValueVT(valuevt) { - Regs.push_back(Reg); + RegsForValue(const TargetLowering &tli, + const SmallVector ®s, + MVT regvt, MVT valuevt) + : TLI(&tli), ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {} + RegsForValue(const TargetLowering &tli, + const SmallVector ®s, + const SmallVector ®vts, + const SmallVector &valuevts) + : TLI(&tli), ValueVTs(valuevts), RegVTs(regvts), Regs(regs) {} + RegsForValue(const TargetLowering &tli, + unsigned Reg, const Type *Ty) : TLI(&tli) { + ComputeValueVTs(tli, Ty, ValueVTs); + + for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { + MVT ValueVT = ValueVTs[Value]; + unsigned NumRegs = TLI->getNumRegisters(ValueVT); + MVT RegisterVT = TLI->getRegisterType(ValueVT); + for (unsigned i = 0; i != NumRegs; ++i) + Regs.push_back(Reg + i); + RegVTs.push_back(RegisterVT); + Reg += NumRegs; + } } - RegsForValue(const std::vector ®s, - MVT::ValueType regvt, MVT::ValueType valuevt) - : Regs(regs), RegVT(regvt), ValueVT(valuevt) { + + /// append - Add the specified values to this one. + void append(const RegsForValue &RHS) { + TLI = RHS.TLI; + ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end()); + RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end()); + Regs.append(RHS.Regs.begin(), RHS.Regs.end()); } + /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from - /// this value and returns the result as a ValueVT value. This uses + /// this value and returns the result as a ValueVTs value. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. - SDOperand getCopyFromRegs(SelectionDAG &DAG, - SDOperand &Chain, SDOperand *Flag) const; + SDValue getCopyFromRegs(SelectionDAG &DAG, + SDValue &Chain, SDValue *Flag) const; /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the /// specified value into the registers specified by this object. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. - void getCopyToRegs(SDOperand Val, SelectionDAG &DAG, - SDOperand &Chain, SDOperand *Flag) const; + void getCopyToRegs(SDValue Val, SelectionDAG &DAG, + SDValue &Chain, SDValue *Flag) const; /// AddInlineAsmOperands - Add this value to the specified inlineasm node /// operand list. This adds the code marker and includes the number of /// values added into it. void AddInlineAsmOperands(unsigned Code, SelectionDAG &DAG, - std::vector &Ops) const; + std::vector &Ops) const; }; } @@ -147,15 +285,16 @@ namespace llvm { /// for the target. ScheduleDAG* createDefaultScheduler(SelectionDAGISel *IS, SelectionDAG *DAG, - MachineBasicBlock *BB) { + MachineBasicBlock *BB, + bool Fast) { TargetLowering &TLI = IS->getTargetLowering(); if (TLI.getSchedulingPreference() == TargetLowering::SchedulingForLatency) { - return createTDListDAGScheduler(IS, DAG, BB); + return createTDListDAGScheduler(IS, DAG, BB, Fast); } else { assert(TLI.getSchedulingPreference() == TargetLowering::SchedulingForRegPressure && "Unknown sched type!"); - return createBURRListDAGScheduler(IS, DAG, BB); + return createBURRListDAGScheduler(IS, DAG, BB, Fast); } } @@ -190,7 +329,7 @@ namespace llvm { SmallSet CatchInfoFound; #endif - unsigned MakeReg(MVT::ValueType VT) { + unsigned MakeReg(MVT VT) { return RegInfo.createVirtualRegister(TLI.getRegClassFor(VT)); } @@ -207,6 +346,16 @@ namespace llvm { assert(R == 0 && "Already initialized this value register!"); return R = CreateRegForValue(V); } + + struct LiveOutInfo { + unsigned NumSignBits; + APInt KnownOne, KnownZero; + LiveOutInfo() : NumSignBits(0) {} + }; + + /// LiveOutRegInfo - Information about live out vregs, indexed by their + /// register number offset by 'FirstVirtualRegister'. + std::vector LiveOutRegInfo; }; } @@ -285,9 +434,9 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, // also creates the initial PHI MachineInstrs, though none of the input // operands are populated. for (BB = Fn.begin(), EB = Fn.end(); BB != EB; ++BB) { - MachineBasicBlock *MBB = new MachineBasicBlock(BB); + MachineBasicBlock *MBB = mf.CreateMachineBasicBlock(BB); MBBMap[BB] = MBB; - MF.getBasicBlockList().push_back(MBB); + MF.push_back(MBB); // Create Machine PHI nodes for LLVM PHI nodes, lowering them as // appropriate. @@ -295,13 +444,19 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, for (BasicBlock::iterator I = BB->begin();(PN = dyn_cast(I)); ++I){ if (PN->use_empty()) continue; - MVT::ValueType VT = TLI.getValueType(PN->getType()); - unsigned NumRegisters = TLI.getNumRegisters(VT); unsigned PHIReg = ValueMap[PN]; assert(PHIReg && "PHI node does not have an assigned virtual register!"); - const TargetInstrInfo *TII = TLI.getTargetMachine().getInstrInfo(); - for (unsigned i = 0; i != NumRegisters; ++i) - BuildMI(MBB, TII->get(TargetInstrInfo::PHI), PHIReg+i); + + SmallVector ValueVTs; + ComputeValueVTs(TLI, PN->getType(), ValueVTs); + for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) { + MVT VT = ValueVTs[vti]; + unsigned NumRegisters = TLI.getNumRegisters(VT); + const TargetInstrInfo *TII = TLI.getTargetMachine().getInstrInfo(); + for (unsigned i = 0; i != NumRegisters; ++i) + BuildMI(MBB, TII->get(TargetInstrInfo::PHI), PHIReg+i); + PHIReg += NumRegisters; + } } } } @@ -309,17 +464,26 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, /// CreateRegForValue - Allocate the appropriate number of virtual registers of /// the correctly promoted or expanded types. Assign these registers /// consecutive vreg numbers and return the first assigned number. +/// +/// In the case that the given value has struct or array type, this function +/// will assign registers for each member or element. +/// unsigned FunctionLoweringInfo::CreateRegForValue(const Value *V) { - MVT::ValueType VT = TLI.getValueType(V->getType()); - - unsigned NumRegisters = TLI.getNumRegisters(VT); - MVT::ValueType RegisterVT = TLI.getRegisterType(VT); + SmallVector ValueVTs; + ComputeValueVTs(TLI, V->getType(), ValueVTs); - unsigned R = MakeReg(RegisterVT); - for (unsigned i = 1; i != NumRegisters; ++i) - MakeReg(RegisterVT); + unsigned FirstReg = 0; + for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { + MVT ValueVT = ValueVTs[Value]; + MVT RegisterVT = TLI.getRegisterType(ValueVT); - return R; + unsigned NumRegs = TLI.getNumRegisters(ValueVT); + for (unsigned i = 0; i != NumRegs; ++i) { + unsigned R = MakeReg(RegisterVT); + if (!FirstReg) FirstReg = R; + } + } + return FirstReg; } //===----------------------------------------------------------------------===// @@ -331,13 +495,20 @@ namespace llvm { class SelectionDAGLowering { MachineBasicBlock *CurMBB; - DenseMap NodeMap; + DenseMap NodeMap; /// PendingLoads - Loads are not emitted to the program immediately. We bunch /// them up and then emit token factor nodes when possible. This allows us to /// get simple disambiguation between loads without worrying about alias /// analysis. - std::vector PendingLoads; + SmallVector PendingLoads; + + /// PendingExports - CopyToReg nodes that copy values to virtual registers + /// for export to other blocks need to be emitted before any terminator + /// instruction, but they have no other ordering requirements. We bunch them + /// up and the emit a single tokenfactor for them just before terminator + /// instructions. + std::vector PendingExports; /// Case - A struct to record the Value for a switch case, and the /// case's target basic block. @@ -440,28 +611,62 @@ public: FuncInfo(funcinfo), GCI(gci) { } - /// getRoot - Return the current virtual root of the Selection DAG. + /// getRoot - Return the current virtual root of the Selection DAG, + /// flushing any PendingLoad items. This must be done before emitting + /// a store or any other node that may need to be ordered after any + /// prior load instructions. /// - SDOperand getRoot() { + SDValue getRoot() { if (PendingLoads.empty()) return DAG.getRoot(); if (PendingLoads.size() == 1) { - SDOperand Root = PendingLoads[0]; + SDValue Root = PendingLoads[0]; DAG.setRoot(Root); PendingLoads.clear(); return Root; } // Otherwise, we have to make a token factor node. - SDOperand Root = DAG.getNode(ISD::TokenFactor, MVT::Other, + SDValue Root = DAG.getNode(ISD::TokenFactor, MVT::Other, &PendingLoads[0], PendingLoads.size()); PendingLoads.clear(); DAG.setRoot(Root); return Root; } - SDOperand CopyValueToVirtualRegister(Value *V, unsigned Reg); + /// getControlRoot - Similar to getRoot, but instead of flushing all the + /// PendingLoad items, flush all the PendingExports items. It is necessary + /// to do this before emitting a terminator instruction. + /// + SDValue getControlRoot() { + SDValue Root = DAG.getRoot(); + + if (PendingExports.empty()) + return Root; + + // Turn all of the CopyToReg chains into one factored node. + if (Root.getOpcode() != ISD::EntryToken) { + unsigned i = 0, e = PendingExports.size(); + for (; i != e; ++i) { + assert(PendingExports[i].Val->getNumOperands() > 1); + if (PendingExports[i].Val->getOperand(0) == Root) + break; // Don't add the root if we already indirectly depend on it. + } + + if (i == e) + PendingExports.push_back(Root); + } + + Root = DAG.getNode(ISD::TokenFactor, MVT::Other, + &PendingExports[0], + PendingExports.size()); + PendingExports.clear(); + DAG.setRoot(Root); + return Root; + } + + void CopyValueToVirtualRegister(Value *V, unsigned Reg); void visit(Instruction &I) { visit(I.getOpcode(), I); } @@ -480,14 +685,10 @@ public: void setCurrentBasicBlock(MachineBasicBlock *MBB) { CurMBB = MBB; } - SDOperand getLoadFrom(const Type *Ty, SDOperand Ptr, - const Value *SV, SDOperand Root, - bool isVolatile, unsigned Alignment); + SDValue getValue(const Value *V); - SDOperand getValue(const Value *V); - - void setValue(const Value *V, SDOperand NewN) { - SDOperand &N = NodeMap[V]; + void setValue(const Value *V, SDValue NewN) { + SDValue &N = NodeMap[V]; assert(N.Val == 0 && "Already set a value for this node!"); N = NewN; } @@ -501,7 +702,7 @@ public: unsigned Opc); bool isExportableFromCurrentBlock(Value *V, const BasicBlock *FromBB); void ExportFromCurrentBlock(Value *V); - void LowerCallTo(CallSite CS, SDOperand Callee, bool IsTailCall, + void LowerCallTo(CallSite CS, SDValue Callee, bool IsTailCall, MachineBasicBlock *LandingPad = NULL); // Terminator instructions. @@ -569,6 +770,8 @@ public: void visitAShr(User &I) { visitShift(I, ISD::SRA); } void visitICmp(User &I); void visitFCmp(User &I); + void visitVICmp(User &I); + void visitVFCmp(User &I); // Visit the conversion instructions void visitTrunc(User &I); void visitZExt(User &I); @@ -587,6 +790,9 @@ public: void visitInsertElement(User &I); void visitShuffleVector(User &I); + void visitExtractValue(ExtractValueInst &I); + void visitInsertValue(InsertValueInst &I); + void visitGetElementPtr(User &I); void visitSelect(User &I); @@ -606,12 +812,6 @@ public: void visitVAEnd(CallInst &I); void visitVACopy(CallInst &I); - void visitMemIntrinsic(CallInst &I, unsigned Op); - - void visitGetResult(GetResultInst &I) { - assert (0 && "getresult unimplemented"); - } - void visitUserOp1(Instruction &I) { assert(0 && "UserOp1 should not exist at instruction selection time!"); abort(); @@ -620,6 +820,10 @@ public: assert(0 && "UserOp2 should not exist at instruction selection time!"); abort(); } + +private: + inline const char *implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op); + }; } // end namespace llvm @@ -629,32 +833,32 @@ public: /// larger then ValueVT then AssertOp can be used to specify whether the extra /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT /// (ISD::AssertSext). -static SDOperand getCopyFromParts(SelectionDAG &DAG, - const SDOperand *Parts, +static SDValue getCopyFromParts(SelectionDAG &DAG, + const SDValue *Parts, unsigned NumParts, - MVT::ValueType PartVT, - MVT::ValueType ValueVT, + MVT PartVT, + MVT ValueVT, ISD::NodeType AssertOp = ISD::DELETED_NODE) { assert(NumParts > 0 && "No parts to assemble!"); TargetLowering &TLI = DAG.getTargetLoweringInfo(); - SDOperand Val = Parts[0]; + SDValue Val = Parts[0]; if (NumParts > 1) { // Assemble the value from multiple parts. - if (!MVT::isVector(ValueVT)) { - unsigned PartBits = MVT::getSizeInBits(PartVT); - unsigned ValueBits = MVT::getSizeInBits(ValueVT); + if (!ValueVT.isVector()) { + unsigned PartBits = PartVT.getSizeInBits(); + unsigned ValueBits = ValueVT.getSizeInBits(); // Assemble the power of 2 part. unsigned RoundParts = NumParts & (NumParts - 1) ? 1 << Log2_32(NumParts) : NumParts; unsigned RoundBits = PartBits * RoundParts; - MVT::ValueType RoundVT = RoundBits == ValueBits ? - ValueVT : MVT::getIntegerType(RoundBits); - SDOperand Lo, Hi; + MVT RoundVT = RoundBits == ValueBits ? + ValueVT : MVT::getIntegerVT(RoundBits); + SDValue Lo, Hi; if (RoundParts > 2) { - MVT::ValueType HalfVT = MVT::getIntegerType(RoundBits/2); + MVT HalfVT = MVT::getIntegerVT(RoundBits/2); Lo = getCopyFromParts(DAG, Parts, RoundParts/2, PartVT, HalfVT); Hi = getCopyFromParts(DAG, Parts+RoundParts/2, RoundParts/2, PartVT, HalfVT); @@ -669,36 +873,36 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, if (RoundParts < NumParts) { // Assemble the trailing non-power-of-2 part. unsigned OddParts = NumParts - RoundParts; - MVT::ValueType OddVT = MVT::getIntegerType(OddParts * PartBits); + MVT OddVT = MVT::getIntegerVT(OddParts * PartBits); Hi = getCopyFromParts(DAG, Parts+RoundParts, OddParts, PartVT, OddVT); // Combine the round and odd parts. Lo = Val; if (TLI.isBigEndian()) std::swap(Lo, Hi); - MVT::ValueType TotalVT = MVT::getIntegerType(NumParts * PartBits); + MVT TotalVT = MVT::getIntegerVT(NumParts * PartBits); Hi = DAG.getNode(ISD::ANY_EXTEND, TotalVT, Hi); Hi = DAG.getNode(ISD::SHL, TotalVT, Hi, - DAG.getConstant(MVT::getSizeInBits(Lo.getValueType()), + DAG.getConstant(Lo.getValueType().getSizeInBits(), TLI.getShiftAmountTy())); Lo = DAG.getNode(ISD::ZERO_EXTEND, TotalVT, Lo); Val = DAG.getNode(ISD::OR, TotalVT, Lo, Hi); } } else { // Handle a multi-element vector. - MVT::ValueType IntermediateVT, RegisterVT; + MVT IntermediateVT, RegisterVT; unsigned NumIntermediates; unsigned NumRegs = TLI.getVectorTypeBreakdown(ValueVT, IntermediateVT, NumIntermediates, RegisterVT); - assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); + NumParts = NumRegs; // Silence a compiler warning. assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); assert(RegisterVT == Parts[0].getValueType() && "Part type doesn't match part!"); // Assemble the parts into intermediate operands. - SmallVector Ops(NumIntermediates); + SmallVector Ops(NumIntermediates); if (NumIntermediates == NumParts) { // If the register was not expanded, truncate or copy the value, // as appropriate. @@ -718,7 +922,7 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the intermediate // operands. - Val = DAG.getNode(MVT::isVector(IntermediateVT) ? + Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR, ValueVT, &Ops[0], NumIntermediates); } @@ -730,21 +934,21 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, if (PartVT == ValueVT) return Val; - if (MVT::isVector(PartVT)) { - assert(MVT::isVector(ValueVT) && "Unknown vector conversion!"); + if (PartVT.isVector()) { + assert(ValueVT.isVector() && "Unknown vector conversion!"); return DAG.getNode(ISD::BIT_CONVERT, ValueVT, Val); } - if (MVT::isVector(ValueVT)) { - assert(MVT::getVectorElementType(ValueVT) == PartVT && - MVT::getVectorNumElements(ValueVT) == 1 && + if (ValueVT.isVector()) { + assert(ValueVT.getVectorElementType() == PartVT && + ValueVT.getVectorNumElements() == 1 && "Only trivial scalar-to-vector conversions should get here!"); return DAG.getNode(ISD::BUILD_VECTOR, ValueVT, Val); } - if (MVT::isInteger(PartVT) && - MVT::isInteger(ValueVT)) { - if (MVT::getSizeInBits(ValueVT) < MVT::getSizeInBits(PartVT)) { + if (PartVT.isInteger() && + ValueVT.isInteger()) { + if (ValueVT.bitsLT(PartVT)) { // For a truncate, see if we have any information to // indicate whether the truncated bits will always be // zero or sign-extension. @@ -757,64 +961,65 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, } } - if (MVT::isFloatingPoint(PartVT) && MVT::isFloatingPoint(ValueVT)) { - if (ValueVT < Val.getValueType()) + if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { + if (ValueVT.bitsLT(Val.getValueType())) // FP_ROUND's are always exact here. return DAG.getNode(ISD::FP_ROUND, ValueVT, Val, DAG.getIntPtrConstant(1)); return DAG.getNode(ISD::FP_EXTEND, ValueVT, Val); } - if (MVT::getSizeInBits(PartVT) == MVT::getSizeInBits(ValueVT)) + if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) return DAG.getNode(ISD::BIT_CONVERT, ValueVT, Val); assert(0 && "Unknown mismatch!"); + return SDValue(); } /// getCopyToParts - Create a series of nodes that contain the specified value /// split into legal parts. If the parts contain more bits than Val, then, for /// integers, ExtendKind can be used to specify how to generate the extra bits. static void getCopyToParts(SelectionDAG &DAG, - SDOperand Val, - SDOperand *Parts, + SDValue Val, + SDValue *Parts, unsigned NumParts, - MVT::ValueType PartVT, + MVT PartVT, ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { TargetLowering &TLI = DAG.getTargetLoweringInfo(); - MVT::ValueType PtrVT = TLI.getPointerTy(); - MVT::ValueType ValueVT = Val.getValueType(); - unsigned PartBits = MVT::getSizeInBits(PartVT); + MVT PtrVT = TLI.getPointerTy(); + MVT ValueVT = Val.getValueType(); + unsigned PartBits = PartVT.getSizeInBits(); assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!"); if (!NumParts) return; - if (!MVT::isVector(ValueVT)) { + if (!ValueVT.isVector()) { if (PartVT == ValueVT) { assert(NumParts == 1 && "No-op copy with multiple parts!"); Parts[0] = Val; return; } - if (NumParts * PartBits > MVT::getSizeInBits(ValueVT)) { + if (NumParts * PartBits > ValueVT.getSizeInBits()) { // If the parts cover more bits than the value has, promote the value. - if (MVT::isFloatingPoint(PartVT) && MVT::isFloatingPoint(ValueVT)) { + if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { assert(NumParts == 1 && "Do not know what to promote to!"); Val = DAG.getNode(ISD::FP_EXTEND, PartVT, Val); - } else if (MVT::isInteger(PartVT) && MVT::isInteger(ValueVT)) { - ValueVT = MVT::getIntegerType(NumParts * PartBits); + } else if (PartVT.isInteger() && ValueVT.isInteger()) { + ValueVT = MVT::getIntegerVT(NumParts * PartBits); Val = DAG.getNode(ExtendKind, ValueVT, Val); } else { assert(0 && "Unknown mismatch!"); } - } else if (PartBits == MVT::getSizeInBits(ValueVT)) { + } else if (PartBits == ValueVT.getSizeInBits()) { // Different types of the same size. assert(NumParts == 1 && PartVT != ValueVT); Val = DAG.getNode(ISD::BIT_CONVERT, PartVT, Val); - } else if (NumParts * PartBits < MVT::getSizeInBits(ValueVT)) { + } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { // If the parts cover less bits than value has, truncate the value. - if (MVT::isInteger(PartVT) && MVT::isInteger(ValueVT)) { - ValueVT = MVT::getIntegerType(NumParts * PartBits); + if (PartVT.isInteger() && ValueVT.isInteger()) { + ValueVT = MVT::getIntegerVT(NumParts * PartBits); Val = DAG.getNode(ISD::TRUNCATE, ValueVT, Val); } else { assert(0 && "Unknown mismatch!"); @@ -823,7 +1028,7 @@ static void getCopyToParts(SelectionDAG &DAG, // The value may have changed - recompute ValueVT. ValueVT = Val.getValueType(); - assert(NumParts * PartBits == MVT::getSizeInBits(ValueVT) && + assert(NumParts * PartBits == ValueVT.getSizeInBits() && "Failed to tile the value with PartVT!"); if (NumParts == 1) { @@ -835,12 +1040,12 @@ static void getCopyToParts(SelectionDAG &DAG, // Expand the value into multiple parts. if (NumParts & (NumParts - 1)) { // The number of parts is not a power of 2. Split off and copy the tail. - assert(MVT::isInteger(PartVT) && MVT::isInteger(ValueVT) && + assert(PartVT.isInteger() && ValueVT.isInteger() && "Do not know what to expand to!"); unsigned RoundParts = 1 << Log2_32(NumParts); unsigned RoundBits = RoundParts * PartBits; unsigned OddParts = NumParts - RoundParts; - SDOperand OddVal = DAG.getNode(ISD::SRL, ValueVT, Val, + SDValue OddVal = DAG.getNode(ISD::SRL, ValueVT, Val, DAG.getConstant(RoundBits, TLI.getShiftAmountTy())); getCopyToParts(DAG, OddVal, Parts + RoundParts, OddParts, PartVT); @@ -848,25 +1053,31 @@ static void getCopyToParts(SelectionDAG &DAG, // The odd parts were reversed by getCopyToParts - unreverse them. std::reverse(Parts + RoundParts, Parts + NumParts); NumParts = RoundParts; - ValueVT = MVT::getIntegerType(NumParts * PartBits); + ValueVT = MVT::getIntegerVT(NumParts * PartBits); Val = DAG.getNode(ISD::TRUNCATE, ValueVT, Val); } // The number of parts is a power of 2. Repeatedly bisect the value using // EXTRACT_ELEMENT. - Parts[0] = Val; + Parts[0] = DAG.getNode(ISD::BIT_CONVERT, + MVT::getIntegerVT(ValueVT.getSizeInBits()), + Val); for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { for (unsigned i = 0; i < NumParts; i += StepSize) { unsigned ThisBits = StepSize * PartBits / 2; - MVT::ValueType ThisVT = - ThisBits == PartBits ? PartVT : MVT::getIntegerType (ThisBits); - - Parts[i+StepSize/2] = - DAG.getNode(ISD::EXTRACT_ELEMENT, ThisVT, Parts[i], - DAG.getConstant(1, PtrVT)); - Parts[i] = - DAG.getNode(ISD::EXTRACT_ELEMENT, ThisVT, Parts[i], - DAG.getConstant(0, PtrVT)); + MVT ThisVT = MVT::getIntegerVT (ThisBits); + SDValue &Part0 = Parts[i]; + SDValue &Part1 = Parts[i+StepSize/2]; + + Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, ThisVT, Part0, + DAG.getConstant(1, PtrVT)); + Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, ThisVT, Part0, + DAG.getConstant(0, PtrVT)); + + if (ThisBits == PartBits && ThisVT != PartVT) { + Part0 = DAG.getNode(ISD::BIT_CONVERT, PartVT, Part0); + Part1 = DAG.getNode(ISD::BIT_CONVERT, PartVT, Part1); + } } } @@ -879,11 +1090,11 @@ static void getCopyToParts(SelectionDAG &DAG, // Vector ValueVT. if (NumParts == 1) { if (PartVT != ValueVT) { - if (MVT::isVector(PartVT)) { + if (PartVT.isVector()) { Val = DAG.getNode(ISD::BIT_CONVERT, PartVT, Val); } else { - assert(MVT::getVectorElementType(ValueVT) == PartVT && - MVT::getVectorNumElements(ValueVT) == 1 && + assert(ValueVT.getVectorElementType() == PartVT && + ValueVT.getVectorNumElements() == 1 && "Only trivial vector-to-scalar conversions should get here!"); Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, PartVT, Val, DAG.getConstant(0, PtrVT)); @@ -895,21 +1106,22 @@ static void getCopyToParts(SelectionDAG &DAG, } // Handle a multi-element vector. - MVT::ValueType IntermediateVT, RegisterVT; + MVT IntermediateVT, RegisterVT; unsigned NumIntermediates; unsigned NumRegs = DAG.getTargetLoweringInfo() .getVectorTypeBreakdown(ValueVT, IntermediateVT, NumIntermediates, RegisterVT); - unsigned NumElements = MVT::getVectorNumElements(ValueVT); + unsigned NumElements = ValueVT.getVectorNumElements(); assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); + NumParts = NumRegs; // Silence a compiler warning. assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); // Split the vector into intermediate operands. - SmallVector Ops(NumIntermediates); + SmallVector Ops(NumIntermediates); for (unsigned i = 0; i != NumIntermediates; ++i) - if (MVT::isVector(IntermediateVT)) + if (IntermediateVT.isVector()) Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, IntermediateVT, Val, DAG.getConstant(i * (NumElements / NumIntermediates), @@ -937,74 +1149,102 @@ static void getCopyToParts(SelectionDAG &DAG, } -SDOperand SelectionDAGLowering::getValue(const Value *V) { - SDOperand &N = NodeMap[V]; +SDValue SelectionDAGLowering::getValue(const Value *V) { + SDValue &N = NodeMap[V]; if (N.Val) return N; - const Type *VTy = V->getType(); - MVT::ValueType VT = TLI.getValueType(VTy); if (Constant *C = const_cast(dyn_cast(V))) { + MVT VT = TLI.getValueType(V->getType(), true); + + if (ConstantInt *CI = dyn_cast(C)) + return N = DAG.getConstant(CI->getValue(), VT); + + if (GlobalValue *GV = dyn_cast(C)) + return N = DAG.getGlobalAddress(GV, VT); + + if (isa(C)) + return N = DAG.getConstant(0, TLI.getPointerTy()); + + if (ConstantFP *CFP = dyn_cast(C)) + return N = DAG.getConstantFP(CFP->getValueAPF(), VT); + + if (isa(C) && !isa(V->getType()) && + !V->getType()->isAggregateType()) + return N = DAG.getNode(ISD::UNDEF, VT); + if (ConstantExpr *CE = dyn_cast(C)) { visit(CE->getOpcode(), *CE); - SDOperand N1 = NodeMap[V]; + SDValue N1 = NodeMap[V]; assert(N1.Val && "visit didn't populate the ValueMap!"); return N1; - } else if (GlobalValue *GV = dyn_cast(C)) { - return N = DAG.getGlobalAddress(GV, VT); - } else if (isa(C)) { - return N = DAG.getConstant(0, TLI.getPointerTy()); - } else if (isa(C)) { - if (!isa(VTy)) - return N = DAG.getNode(ISD::UNDEF, VT); - - // Create a BUILD_VECTOR of undef nodes. - const VectorType *PTy = cast(VTy); - unsigned NumElements = PTy->getNumElements(); - MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); + } + + if (isa(C) || isa(C)) { + SmallVector Constants; + for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end(); + OI != OE; ++OI) { + SDNode *Val = getValue(*OI).Val; + for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) + Constants.push_back(SDValue(Val, i)); + } + return DAG.getMergeValues(&Constants[0], Constants.size()); + } - SmallVector Ops; - Ops.assign(NumElements, DAG.getNode(ISD::UNDEF, PVT)); - - // Create a VConstant node with generic Vector type. - MVT::ValueType VT = MVT::getVectorType(PVT, NumElements); - return N = DAG.getNode(ISD::BUILD_VECTOR, VT, - &Ops[0], Ops.size()); - } else if (ConstantFP *CFP = dyn_cast(C)) { - return N = DAG.getConstantFP(CFP->getValueAPF(), VT); - } else if (const VectorType *PTy = dyn_cast(VTy)) { - unsigned NumElements = PTy->getNumElements(); - MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); - - // Now that we know the number and type of the elements, push a - // Constant or ConstantFP node onto the ops list for each element of - // the vector constant. - SmallVector Ops; - if (ConstantVector *CP = dyn_cast(C)) { - for (unsigned i = 0; i != NumElements; ++i) - Ops.push_back(getValue(CP->getOperand(i))); - } else { - assert(isa(C) && "Unknown vector constant!"); - SDOperand Op; - if (MVT::isFloatingPoint(PVT)) - Op = DAG.getConstantFP(0, PVT); + if (isa(C->getType()) || isa(C->getType())) { + assert((isa(C) || isa(C)) && + "Unknown struct or array constant!"); + + SmallVector ValueVTs; + ComputeValueVTs(TLI, C->getType(), ValueVTs); + unsigned NumElts = ValueVTs.size(); + if (NumElts == 0) + return SDValue(); // empty struct + SmallVector Constants(NumElts); + for (unsigned i = 0; i != NumElts; ++i) { + MVT EltVT = ValueVTs[i]; + if (isa(C)) + Constants[i] = DAG.getNode(ISD::UNDEF, EltVT); + else if (EltVT.isFloatingPoint()) + Constants[i] = DAG.getConstantFP(0, EltVT); else - Op = DAG.getConstant(0, PVT); - Ops.assign(NumElements, Op); + Constants[i] = DAG.getConstant(0, EltVT); } - - // Create a BUILD_VECTOR node. - MVT::ValueType VT = MVT::getVectorType(PVT, NumElements); - return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], - Ops.size()); + return DAG.getMergeValues(&Constants[0], NumElts); + } + + const VectorType *VecTy = cast(V->getType()); + unsigned NumElements = VecTy->getNumElements(); + + // Now that we know the number and type of the elements, get that number of + // elements into the Ops array based on what kind of constant it is. + SmallVector Ops; + if (ConstantVector *CP = dyn_cast(C)) { + for (unsigned i = 0; i != NumElements; ++i) + Ops.push_back(getValue(CP->getOperand(i))); } else { - // Canonicalize all constant ints to be unsigned. - return N = DAG.getConstant(cast(C)->getValue(),VT); + assert((isa(C) || isa(C)) && + "Unknown vector constant!"); + MVT EltVT = TLI.getValueType(VecTy->getElementType()); + + SDValue Op; + if (isa(C)) + Op = DAG.getNode(ISD::UNDEF, EltVT); + else if (EltVT.isFloatingPoint()) + Op = DAG.getConstantFP(0, EltVT); + else + Op = DAG.getConstant(0, EltVT); + Ops.assign(NumElements, Op); } + + // Create a BUILD_VECTOR node. + return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } + // If this is a static alloca, generate it as the frameindex instead of + // computation. if (const AllocaInst *AI = dyn_cast(V)) { std::map::iterator SI = - FuncInfo.StaticAllocaMap.find(AI); + FuncInfo.StaticAllocaMap.find(AI); if (SI != FuncInfo.StaticAllocaMap.end()) return DAG.getFrameIndex(SI->second, TLI.getPointerTy()); } @@ -1012,55 +1252,54 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { unsigned InReg = FuncInfo.ValueMap[V]; assert(InReg && "Value not in map!"); - MVT::ValueType RegisterVT = TLI.getRegisterType(VT); - unsigned NumRegs = TLI.getNumRegisters(VT); - - std::vector Regs(NumRegs); - for (unsigned i = 0; i != NumRegs; ++i) - Regs[i] = InReg + i; - - RegsForValue RFV(Regs, RegisterVT, VT); - SDOperand Chain = DAG.getEntryNode(); - + RegsForValue RFV(TLI, InReg, V->getType()); + SDValue Chain = DAG.getEntryNode(); return RFV.getCopyFromRegs(DAG, Chain, NULL); } void SelectionDAGLowering::visitRet(ReturnInst &I) { if (I.getNumOperands() == 0) { - DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, getRoot())); + DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, getControlRoot())); return; } - SmallVector NewValues; - NewValues.push_back(getRoot()); - for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { - SDOperand RetOp = getValue(I.getOperand(i)); - MVT::ValueType VT = RetOp.getValueType(); - - // FIXME: C calling convention requires the return type to be promoted to - // at least 32-bit. But this is not necessary for non-C calling conventions. - if (MVT::isInteger(VT)) { - MVT::ValueType MinVT = TLI.getRegisterType(MVT::i32); - if (MVT::getSizeInBits(VT) < MVT::getSizeInBits(MinVT)) - VT = MinVT; - } - - unsigned NumParts = TLI.getNumRegisters(VT); - MVT::ValueType PartVT = TLI.getRegisterType(VT); - SmallVector Parts(NumParts); - ISD::NodeType ExtendKind = ISD::ANY_EXTEND; - - const Function *F = I.getParent()->getParent(); - if (F->paramHasAttr(0, ParamAttr::SExt)) - ExtendKind = ISD::SIGN_EXTEND; - else if (F->paramHasAttr(0, ParamAttr::ZExt)) - ExtendKind = ISD::ZERO_EXTEND; - - getCopyToParts(DAG, RetOp, &Parts[0], NumParts, PartVT, ExtendKind); + + SmallVector NewValues; + NewValues.push_back(getControlRoot()); + for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { + SDValue RetOp = getValue(I.getOperand(i)); + + SmallVector ValueVTs; + ComputeValueVTs(TLI, I.getOperand(i)->getType(), ValueVTs); + for (unsigned j = 0, f = ValueVTs.size(); j != f; ++j) { + MVT VT = ValueVTs[j]; + + // FIXME: C calling convention requires the return type to be promoted to + // at least 32-bit. But this is not necessary for non-C calling conventions. + if (VT.isInteger()) { + MVT MinVT = TLI.getRegisterType(MVT::i32); + if (VT.bitsLT(MinVT)) + VT = MinVT; + } - for (unsigned i = 0; i < NumParts; ++i) { - NewValues.push_back(Parts[i]); - NewValues.push_back(DAG.getConstant(false, MVT::i32)); + unsigned NumParts = TLI.getNumRegisters(VT); + MVT PartVT = TLI.getRegisterType(VT); + SmallVector Parts(NumParts); + ISD::NodeType ExtendKind = ISD::ANY_EXTEND; + + const Function *F = I.getParent()->getParent(); + if (F->paramHasAttr(0, ParamAttr::SExt)) + ExtendKind = ISD::SIGN_EXTEND; + else if (F->paramHasAttr(0, ParamAttr::ZExt)) + ExtendKind = ISD::ZERO_EXTEND; + + getCopyToParts(DAG, SDValue(RetOp.Val, RetOp.ResNo + j), + &Parts[0], NumParts, PartVT, ExtendKind); + + for (unsigned i = 0; i < NumParts; ++i) { + NewValues.push_back(Parts[i]); + NewValues.push_back(DAG.getArgFlags(ISD::ArgFlagsTy())); + } } } DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, @@ -1078,7 +1317,7 @@ void SelectionDAGLowering::ExportFromCurrentBlock(Value *V) { if (FuncInfo.isExportedInst(V)) return; unsigned Reg = FuncInfo.InitializeRegForValue(V); - PendingLoads.push_back(CopyValueToVirtualRegister(V, Reg)); + CopyValueToVirtualRegister(V, Reg); } bool SelectionDAGLowering::isExportableFromCurrentBlock(Value *V, @@ -1166,8 +1405,8 @@ void SelectionDAGLowering::FindMergedConditions(Value *Cond, case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break; case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break; case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break; - case FCmpInst::FCMP_ORD: FOC = ISD::SETEQ; FPC = ISD::SETO; break; - case FCmpInst::FCMP_UNO: FOC = ISD::SETNE; FPC = ISD::SETUO; break; + case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break; + case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break; case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break; case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break; case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break; @@ -1201,8 +1440,9 @@ void SelectionDAGLowering::FindMergedConditions(Value *Cond, // Create TmpBB after CurBB. MachineFunction::iterator BBI = CurBB; - MachineBasicBlock *TmpBB = new MachineBasicBlock(CurBB->getBasicBlock()); - CurBB->getParent()->getBasicBlockList().insert(++BBI, TmpBB); + MachineFunction &MF = DAG.getMachineFunction(); + MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock()); + CurBB->getParent()->insert(++BBI, TmpBB); if (Opc == Instruction::Or) { // Codegen X | Y as: @@ -1267,13 +1507,13 @@ void SelectionDAGLowering::visitBr(BranchInst &I) { NextBlock = BBI; if (I.isUnconditional()) { + // Update machine-CFG edges. + CurMBB->addSuccessor(Succ0MBB); + // If this is not a fall-through branch, emit the branch. if (Succ0MBB != NextBlock) - DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getRoot(), + DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getControlRoot(), DAG.getBasicBlock(Succ0MBB))); - - // Update machine-CFG edges. - CurMBB->addSuccessor(Succ0MBB); return; } @@ -1323,7 +1563,7 @@ void SelectionDAGLowering::visitBr(BranchInst &I) { // Okay, we decided not to do this, remove any inserted MBB's and clear // SwitchCases. for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) - CurMBB->getParent()->getBasicBlockList().erase(SwitchCases[i].ThisBB); + CurMBB->getParent()->erase(SwitchCases[i].ThisBB); SwitchCases.clear(); } @@ -1340,8 +1580,8 @@ void SelectionDAGLowering::visitBr(BranchInst &I) { /// visitSwitchCase - Emits the necessary code to represent a single node in /// the binary search tree resulting from lowering a switch instruction. void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) { - SDOperand Cond; - SDOperand CondLHS = getValue(CB.CmpLHS); + SDValue Cond; + SDValue CondLHS = getValue(CB.CmpLHS); // Build the setcc now. if (CB.CmpMHS == NULL) { @@ -1350,7 +1590,7 @@ void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) { if (CB.CmpRHS == ConstantInt::getTrue() && CB.CC == ISD::SETEQ) Cond = CondLHS; else if (CB.CmpRHS == ConstantInt::getFalse() && CB.CC == ISD::SETEQ) { - SDOperand True = DAG.getConstant(1, CondLHS.getValueType()); + SDValue True = DAG.getConstant(1, CondLHS.getValueType()); Cond = DAG.getNode(ISD::XOR, CondLHS.getValueType(), CondLHS, True); } else Cond = DAG.getSetCC(MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC); @@ -1360,19 +1600,22 @@ void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) { uint64_t Low = cast(CB.CmpLHS)->getSExtValue(); uint64_t High = cast(CB.CmpRHS)->getSExtValue(); - SDOperand CmpOp = getValue(CB.CmpMHS); - MVT::ValueType VT = CmpOp.getValueType(); + SDValue CmpOp = getValue(CB.CmpMHS); + MVT VT = CmpOp.getValueType(); if (cast(CB.CmpLHS)->isMinValue(true)) { Cond = DAG.getSetCC(MVT::i1, CmpOp, DAG.getConstant(High, VT), ISD::SETLE); } else { - SDOperand SUB = DAG.getNode(ISD::SUB, VT, CmpOp, DAG.getConstant(Low, VT)); + SDValue SUB = DAG.getNode(ISD::SUB, VT, CmpOp, DAG.getConstant(Low, VT)); Cond = DAG.getSetCC(MVT::i1, SUB, DAG.getConstant(High-Low, VT), ISD::SETULE); } - } + // Update successor info + CurMBB->addSuccessor(CB.TrueBB); + CurMBB->addSuccessor(CB.FalseBB); + // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; @@ -1384,28 +1627,38 @@ void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) { // fall through to the lhs instead of the rhs block. if (CB.TrueBB == NextBlock) { std::swap(CB.TrueBB, CB.FalseBB); - SDOperand True = DAG.getConstant(1, Cond.getValueType()); + SDValue True = DAG.getConstant(1, Cond.getValueType()); Cond = DAG.getNode(ISD::XOR, Cond.getValueType(), Cond, True); } - SDOperand BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, getRoot(), Cond, + SDValue BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, getControlRoot(), Cond, DAG.getBasicBlock(CB.TrueBB)); - if (CB.FalseBB == NextBlock) + + // If the branch was constant folded, fix up the CFG. + if (BrCond.getOpcode() == ISD::BR) { + if (!DisableCorrectBranchFolding) + CurMBB->removeSuccessor(CB.FalseBB); DAG.setRoot(BrCond); - else - DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, BrCond, - DAG.getBasicBlock(CB.FalseBB))); - // Update successor info - CurMBB->addSuccessor(CB.TrueBB); - CurMBB->addSuccessor(CB.FalseBB); + } else { + // Otherwise, go ahead and insert the false branch. + if (BrCond == getControlRoot()) + if (!DisableCorrectBranchFolding) + CurMBB->removeSuccessor(CB.TrueBB); + + if (CB.FalseBB == NextBlock) + DAG.setRoot(BrCond); + else + DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, BrCond, + DAG.getBasicBlock(CB.FalseBB))); + } } /// visitJumpTable - Emit JumpTable node in the current MBB void SelectionDAGLowering::visitJumpTable(SelectionDAGISel::JumpTable &JT) { // Emit the code for the jump table assert(JT.Reg != -1U && "Should lower JT Header first!"); - MVT::ValueType PTy = TLI.getPointerTy(); - SDOperand Index = DAG.getCopyFromReg(getRoot(), JT.Reg, PTy); - SDOperand Table = DAG.getJumpTable(JT.JTI, PTy); + MVT PTy = TLI.getPointerTy(); + SDValue Index = DAG.getCopyFromReg(getControlRoot(), JT.Reg, PTy); + SDValue Table = DAG.getJumpTable(JT.JTI, PTy); DAG.setRoot(DAG.getNode(ISD::BR_JT, MVT::Other, Index.getValue(1), Table, Index)); return; @@ -1418,9 +1671,9 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, // Subtract the lowest switch case value from the value being switched on // and conditional branch to default mbb if the result is greater than the // difference between smallest and largest cases. - SDOperand SwitchOp = getValue(JTH.SValue); - MVT::ValueType VT = SwitchOp.getValueType(); - SDOperand SUB = DAG.getNode(ISD::SUB, VT, SwitchOp, + SDValue SwitchOp = getValue(JTH.SValue); + MVT VT = SwitchOp.getValueType(); + SDValue SUB = DAG.getNode(ISD::SUB, VT, SwitchOp, DAG.getConstant(JTH.First, VT)); // The SDNode we just created, which holds the value being switched on @@ -1428,19 +1681,19 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, // register so it can be used as an index into the jump table in a // subsequent basic block. This value may be smaller or larger than the // target's pointer type, and therefore require extension or truncating. - if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy())) + if (VT.bitsGT(TLI.getPointerTy())) SwitchOp = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), SUB); else SwitchOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), SUB); unsigned JumpTableReg = FuncInfo.MakeReg(TLI.getPointerTy()); - SDOperand CopyTo = DAG.getCopyToReg(getRoot(), JumpTableReg, SwitchOp); + SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), JumpTableReg, SwitchOp); JT.Reg = JumpTableReg; // Emit the range check for the jump table, and branch to the default // block for the switch statement if the value being switched on exceeds // the largest case in the switch. - SDOperand CMP = DAG.getSetCC(TLI.getSetCCResultTy(), SUB, + SDValue CMP = DAG.getSetCC(TLI.getSetCCResultType(SUB), SUB, DAG.getConstant(JTH.Last-JTH.First,VT), ISD::SETUGT); @@ -1451,7 +1704,7 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, if (++BBI != CurMBB->getParent()->end()) NextBlock = BBI; - SDOperand BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, CMP, + SDValue BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, CMP, DAG.getBasicBlock(JT.Default)); if (JT.MBB == NextBlock) @@ -1467,34 +1720,31 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, /// suitable for "bit tests" void SelectionDAGLowering::visitBitTestHeader(SelectionDAGISel::BitTestBlock &B) { // Subtract the minimum value - SDOperand SwitchOp = getValue(B.SValue); - MVT::ValueType VT = SwitchOp.getValueType(); - SDOperand SUB = DAG.getNode(ISD::SUB, VT, SwitchOp, + SDValue SwitchOp = getValue(B.SValue); + MVT VT = SwitchOp.getValueType(); + SDValue SUB = DAG.getNode(ISD::SUB, VT, SwitchOp, DAG.getConstant(B.First, VT)); // Check range - SDOperand RangeCmp = DAG.getSetCC(TLI.getSetCCResultTy(), SUB, + SDValue RangeCmp = DAG.getSetCC(TLI.getSetCCResultType(SUB), SUB, DAG.getConstant(B.Range, VT), ISD::SETUGT); - SDOperand ShiftOp; - if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getShiftAmountTy())) + SDValue ShiftOp; + if (VT.bitsGT(TLI.getShiftAmountTy())) ShiftOp = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), SUB); else ShiftOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getShiftAmountTy(), SUB); // Make desired shift - SDOperand SwitchVal = DAG.getNode(ISD::SHL, TLI.getPointerTy(), + SDValue SwitchVal = DAG.getNode(ISD::SHL, TLI.getPointerTy(), DAG.getConstant(1, TLI.getPointerTy()), ShiftOp); unsigned SwitchReg = FuncInfo.MakeReg(TLI.getPointerTy()); - SDOperand CopyTo = DAG.getCopyToReg(getRoot(), SwitchReg, SwitchVal); + SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), SwitchReg, SwitchVal); B.Reg = SwitchReg; - SDOperand BrRange = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, RangeCmp, - DAG.getBasicBlock(B.Default)); - // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; @@ -1503,15 +1753,19 @@ void SelectionDAGLowering::visitBitTestHeader(SelectionDAGISel::BitTestBlock &B) NextBlock = BBI; MachineBasicBlock* MBB = B.Cases[0].ThisBB; + + CurMBB->addSuccessor(B.Default); + CurMBB->addSuccessor(MBB); + + SDValue BrRange = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, RangeCmp, + DAG.getBasicBlock(B.Default)); + if (MBB == NextBlock) DAG.setRoot(BrRange); else DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, CopyTo, DAG.getBasicBlock(MBB))); - CurMBB->addSuccessor(B.Default); - CurMBB->addSuccessor(MBB); - return; } @@ -1520,16 +1774,19 @@ void SelectionDAGLowering::visitBitTestCase(MachineBasicBlock* NextMBB, unsigned Reg, SelectionDAGISel::BitTestCase &B) { // Emit bit tests and jumps - SDOperand SwitchVal = DAG.getCopyFromReg(getRoot(), Reg, TLI.getPointerTy()); + SDValue SwitchVal = DAG.getCopyFromReg(getControlRoot(), Reg, + TLI.getPointerTy()); - SDOperand AndOp = DAG.getNode(ISD::AND, TLI.getPointerTy(), - SwitchVal, - DAG.getConstant(B.Mask, - TLI.getPointerTy())); - SDOperand AndCmp = DAG.getSetCC(TLI.getSetCCResultTy(), AndOp, + SDValue AndOp = DAG.getNode(ISD::AND, TLI.getPointerTy(), SwitchVal, + DAG.getConstant(B.Mask, TLI.getPointerTy())); + SDValue AndCmp = DAG.getSetCC(TLI.getSetCCResultType(AndOp), AndOp, DAG.getConstant(0, TLI.getPointerTy()), ISD::SETNE); - SDOperand BrAnd = DAG.getNode(ISD::BRCOND, MVT::Other, getRoot(), + + CurMBB->addSuccessor(B.TargetBB); + CurMBB->addSuccessor(NextMBB); + + SDValue BrAnd = DAG.getNode(ISD::BRCOND, MVT::Other, getControlRoot(), AndCmp, DAG.getBasicBlock(B.TargetBB)); // Set NextBlock to be the MBB immediately after the current one, if any. @@ -1545,9 +1802,6 @@ void SelectionDAGLowering::visitBitTestCase(MachineBasicBlock* NextMBB, DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, BrAnd, DAG.getBasicBlock(NextMBB))); - CurMBB->addSuccessor(B.TargetBB); - CurMBB->addSuccessor(NextMBB); - return; } @@ -1566,16 +1820,16 @@ void SelectionDAGLowering::visitInvoke(InvokeInst &I) { if (!I.use_empty()) { DenseMap::iterator VMI = FuncInfo.ValueMap.find(&I); if (VMI != FuncInfo.ValueMap.end()) - DAG.setRoot(CopyValueToVirtualRegister(&I, VMI->second)); + CopyValueToVirtualRegister(&I, VMI->second); } - // Drop into normal successor. - DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getRoot(), - DAG.getBasicBlock(Return))); - // Update successor info CurMBB->addSuccessor(Return); CurMBB->addSuccessor(LandingPad); + + // Drop into normal successor. + DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getControlRoot(), + DAG.getBasicBlock(Return))); } void SelectionDAGLowering::visitUnwind(UnwindInst &I) { @@ -1629,8 +1883,8 @@ bool SelectionDAGLowering::handleSmallSwitchRange(CaseRec& CR, for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) { MachineBasicBlock *FallThrough; if (I != E-1) { - FallThrough = new MachineBasicBlock(CurBlock->getBasicBlock()); - CurMF->getBasicBlockList().insert(BBI, FallThrough); + FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock()); + CurMF->insert(BBI, FallThrough); } else { // If the last case doesn't match, go to the default block. FallThrough = Default; @@ -1665,8 +1919,9 @@ bool SelectionDAGLowering::handleSmallSwitchRange(CaseRec& CR, } static inline bool areJTsAllowed(const TargetLowering &TLI) { - return (TLI.isOperationLegal(ISD::BR_JT, MVT::Other) || - TLI.isOperationLegal(ISD::BRIND, MVT::Other)); + return !DisableJumpTables && + (TLI.isOperationLegal(ISD::BR_JT, MVT::Other) || + TLI.isOperationLegal(ISD::BRIND, MVT::Other)); } /// handleJTSwitchCase - Emit jumptable for current switch case range @@ -1713,8 +1968,8 @@ bool SelectionDAGLowering::handleJTSwitchCase(CaseRec& CR, // of the jump table, and jumping to it. Update successor information; // we will either branch to the default case for the switch, or the jump // table. - MachineBasicBlock *JumpTableBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, JumpTableBB); + MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, JumpTableBB); CR.CaseBB->addSuccessor(Default); CR.CaseBB->addSuccessor(JumpTableBB); @@ -1851,8 +2106,8 @@ bool SelectionDAGLowering::handleBTSplitSwitchCase(CaseRec& CR, (cast(CR.GE)->getSExtValue() + 1LL)) { TrueBB = LHSR.first->BB; } else { - TrueBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, TrueBB); + TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, TrueBB); WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR)); } @@ -1865,8 +2120,8 @@ bool SelectionDAGLowering::handleBTSplitSwitchCase(CaseRec& CR, (cast(CR.LT)->getSExtValue() - 1LL)) { FalseBB = RHSR.first->BB; } else { - FalseBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, FalseBB); + FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, FalseBB); WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR)); } @@ -1891,7 +2146,7 @@ bool SelectionDAGLowering::handleBitTestsSwitchCase(CaseRec& CR, CaseRecVector& WorkList, Value* SV, MachineBasicBlock* Default){ - unsigned IntPtrBits = MVT::getSizeInBits(TLI.getPointerTy()); + unsigned IntPtrBits = TLI.getPointerTy().getSizeInBits(); Case& FrontCase = *CR.Range.first; Case& BackCase = *(CR.Range.second-1); @@ -1988,8 +2243,8 @@ bool SelectionDAGLowering::handleBitTestsSwitchCase(CaseRec& CR, DOUT << "Mask: " << CasesBits[i].Mask << ", Bits: " << CasesBits[i].Bits << ", BB: " << CasesBits[i].BB << "\n"; - MachineBasicBlock *CaseBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, CaseBB); + MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, CaseBB); BTC.push_back(SelectionDAGISel::BitTestCase(CasesBits[i].Mask, CaseBB, CasesBits[i].BB)); @@ -2008,7 +2263,7 @@ bool SelectionDAGLowering::handleBitTestsSwitchCase(CaseRec& CR, } -// Clusterify - Transform simple list of Cases into list of CaseRange's +/// Clusterify - Transform simple list of Cases into list of CaseRange's unsigned SelectionDAGLowering::Clusterify(CaseVector& Cases, const SwitchInst& SI) { unsigned numCmps = 0; @@ -2064,11 +2319,11 @@ void SelectionDAGLowering::visitSwitch(SwitchInst &SI) { // Update machine-CFG edges. // If this is not a fall-through branch, emit the branch. + CurMBB->addSuccessor(Default); if (Default != NextBlock) - DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getRoot(), + DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getControlRoot(), DAG.getBasicBlock(Default))); - - CurMBB->addSuccessor(Default); + return; } @@ -2127,7 +2382,7 @@ void SelectionDAGLowering::visitSub(User &I) { std::vector NZ(VL, ConstantFP::getNegativeZero(ElTy)); Constant *CNZ = ConstantVector::get(&NZ[0], NZ.size()); if (CV == CNZ) { - SDOperand Op2 = getValue(I.getOperand(1)); + SDValue Op2 = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2)); return; } @@ -2137,7 +2392,7 @@ void SelectionDAGLowering::visitSub(User &I) { if (Ty->isFloatingPoint()) { if (ConstantFP *CFP = dyn_cast(I.getOperand(0))) if (CFP->isExactlyValue(ConstantFP::getNegativeZero(Ty)->getValueAPF())) { - SDOperand Op2 = getValue(I.getOperand(1)); + SDValue Op2 = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2)); return; } @@ -2147,21 +2402,21 @@ void SelectionDAGLowering::visitSub(User &I) { } void SelectionDAGLowering::visitBinary(User &I, unsigned OpCode) { - SDOperand Op1 = getValue(I.getOperand(0)); - SDOperand Op2 = getValue(I.getOperand(1)); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(OpCode, Op1.getValueType(), Op1, Op2)); } void SelectionDAGLowering::visitShift(User &I, unsigned Opcode) { - SDOperand Op1 = getValue(I.getOperand(0)); - SDOperand Op2 = getValue(I.getOperand(1)); - - if (MVT::getSizeInBits(TLI.getShiftAmountTy()) < - MVT::getSizeInBits(Op2.getValueType())) - Op2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Op2); - else if (TLI.getShiftAmountTy() > Op2.getValueType()) - Op2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Op2); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); + if (!isa(I.getType())) { + if (TLI.getShiftAmountTy().bitsLT(Op2.getValueType())) + Op2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Op2); + else if (TLI.getShiftAmountTy().bitsGT(Op2.getValueType())) + Op2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Op2); + } setValue(&I, DAG.getNode(Opcode, Op1.getValueType(), Op1, Op2)); } @@ -2172,8 +2427,8 @@ void SelectionDAGLowering::visitICmp(User &I) { predicate = IC->getPredicate(); else if (ConstantExpr *IC = dyn_cast(&I)) predicate = ICmpInst::Predicate(IC->getPredicate()); - SDOperand Op1 = getValue(I.getOperand(0)); - SDOperand Op2 = getValue(I.getOperand(1)); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); ISD::CondCode Opcode; switch (predicate) { case ICmpInst::ICMP_EQ : Opcode = ISD::SETEQ; break; @@ -2200,8 +2455,8 @@ void SelectionDAGLowering::visitFCmp(User &I) { predicate = FC->getPredicate(); else if (ConstantExpr *FC = dyn_cast(&I)) predicate = FCmpInst::Predicate(FC->getPredicate()); - SDOperand Op1 = getValue(I.getOperand(0)); - SDOperand Op2 = getValue(I.getOperand(1)); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); ISD::CondCode Condition, FOC, FPC; switch (predicate) { case FCmpInst::FCMP_FALSE: FOC = FPC = ISD::SETFALSE; break; @@ -2211,8 +2466,8 @@ void SelectionDAGLowering::visitFCmp(User &I) { case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break; case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break; case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break; - case FCmpInst::FCMP_ORD: FOC = ISD::SETEQ; FPC = ISD::SETO; break; - case FCmpInst::FCMP_UNO: FOC = ISD::SETNE; FPC = ISD::SETUO; break; + case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break; + case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break; case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break; case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break; case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break; @@ -2232,10 +2487,79 @@ void SelectionDAGLowering::visitFCmp(User &I) { setValue(&I, DAG.getSetCC(MVT::i1, Op1, Op2, Condition)); } +void SelectionDAGLowering::visitVICmp(User &I) { + ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE; + if (VICmpInst *IC = dyn_cast(&I)) + predicate = IC->getPredicate(); + else if (ConstantExpr *IC = dyn_cast(&I)) + predicate = ICmpInst::Predicate(IC->getPredicate()); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); + ISD::CondCode Opcode; + switch (predicate) { + case ICmpInst::ICMP_EQ : Opcode = ISD::SETEQ; break; + case ICmpInst::ICMP_NE : Opcode = ISD::SETNE; break; + case ICmpInst::ICMP_UGT : Opcode = ISD::SETUGT; break; + case ICmpInst::ICMP_UGE : Opcode = ISD::SETUGE; break; + case ICmpInst::ICMP_ULT : Opcode = ISD::SETULT; break; + case ICmpInst::ICMP_ULE : Opcode = ISD::SETULE; break; + case ICmpInst::ICMP_SGT : Opcode = ISD::SETGT; break; + case ICmpInst::ICMP_SGE : Opcode = ISD::SETGE; break; + case ICmpInst::ICMP_SLT : Opcode = ISD::SETLT; break; + case ICmpInst::ICMP_SLE : Opcode = ISD::SETLE; break; + default: + assert(!"Invalid ICmp predicate value"); + Opcode = ISD::SETEQ; + break; + } + setValue(&I, DAG.getVSetCC(Op1.getValueType(), Op1, Op2, Opcode)); +} + +void SelectionDAGLowering::visitVFCmp(User &I) { + FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE; + if (VFCmpInst *FC = dyn_cast(&I)) + predicate = FC->getPredicate(); + else if (ConstantExpr *FC = dyn_cast(&I)) + predicate = FCmpInst::Predicate(FC->getPredicate()); + SDValue Op1 = getValue(I.getOperand(0)); + SDValue Op2 = getValue(I.getOperand(1)); + ISD::CondCode Condition, FOC, FPC; + switch (predicate) { + case FCmpInst::FCMP_FALSE: FOC = FPC = ISD::SETFALSE; break; + case FCmpInst::FCMP_OEQ: FOC = ISD::SETEQ; FPC = ISD::SETOEQ; break; + case FCmpInst::FCMP_OGT: FOC = ISD::SETGT; FPC = ISD::SETOGT; break; + case FCmpInst::FCMP_OGE: FOC = ISD::SETGE; FPC = ISD::SETOGE; break; + case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break; + case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break; + case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break; + case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break; + case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break; + case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break; + case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break; + case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break; + case FCmpInst::FCMP_ULT: FOC = ISD::SETLT; FPC = ISD::SETULT; break; + case FCmpInst::FCMP_ULE: FOC = ISD::SETLE; FPC = ISD::SETULE; break; + case FCmpInst::FCMP_UNE: FOC = ISD::SETNE; FPC = ISD::SETUNE; break; + case FCmpInst::FCMP_TRUE: FOC = FPC = ISD::SETTRUE; break; + default: + assert(!"Invalid VFCmp predicate value"); + FOC = FPC = ISD::SETFALSE; + break; + } + if (FiniteOnlyFPMath()) + Condition = FOC; + else + Condition = FPC; + + MVT DestVT = TLI.getValueType(I.getType()); + + setValue(&I, DAG.getVSetCC(DestVT, Op1, Op2, Condition)); +} + void SelectionDAGLowering::visitSelect(User &I) { - SDOperand Cond = getValue(I.getOperand(0)); - SDOperand TrueVal = getValue(I.getOperand(1)); - SDOperand FalseVal = getValue(I.getOperand(2)); + SDValue Cond = getValue(I.getOperand(0)); + SDValue TrueVal = getValue(I.getOperand(1)); + SDValue FalseVal = getValue(I.getOperand(2)); setValue(&I, DAG.getNode(ISD::SELECT, TrueVal.getValueType(), Cond, TrueVal, FalseVal)); } @@ -2243,77 +2567,77 @@ void SelectionDAGLowering::visitSelect(User &I) { void SelectionDAGLowering::visitTrunc(User &I) { // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::TRUNCATE, DestVT, N)); } void SelectionDAGLowering::visitZExt(User &I) { // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). // ZExt also can't be a cast to bool for same reason. So, nothing much to do - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, DestVT, N)); } void SelectionDAGLowering::visitSExt(User &I) { // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). // SExt also can't be a cast to bool for same reason. So, nothing much to do - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, DestVT, N)); } void SelectionDAGLowering::visitFPTrunc(User &I) { // FPTrunc is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_ROUND, DestVT, N, DAG.getIntPtrConstant(0))); } void SelectionDAGLowering::visitFPExt(User &I){ // FPTrunc is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_EXTEND, DestVT, N)); } void SelectionDAGLowering::visitFPToUI(User &I) { // FPToUI is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_TO_UINT, DestVT, N)); } void SelectionDAGLowering::visitFPToSI(User &I) { // FPToSI is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_TO_SINT, DestVT, N)); } void SelectionDAGLowering::visitUIToFP(User &I) { // UIToFP is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::UINT_TO_FP, DestVT, N)); } void SelectionDAGLowering::visitSIToFP(User &I){ // UIToFP is never a no-op cast, no need to check - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::SINT_TO_FP, DestVT, N)); } void SelectionDAGLowering::visitPtrToInt(User &I) { // What to do depends on the size of the integer and the size of the pointer. // We can either truncate, zero extend, or no-op, accordingly. - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType SrcVT = N.getValueType(); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); - SDOperand Result; - if (MVT::getSizeInBits(DestVT) < MVT::getSizeInBits(SrcVT)) + SDValue N = getValue(I.getOperand(0)); + MVT SrcVT = N.getValueType(); + MVT DestVT = TLI.getValueType(I.getType()); + SDValue Result; + if (DestVT.bitsLT(SrcVT)) Result = DAG.getNode(ISD::TRUNCATE, DestVT, N); else // Note: ZERO_EXTEND can handle cases where the sizes are equal too @@ -2324,10 +2648,10 @@ void SelectionDAGLowering::visitPtrToInt(User &I) { void SelectionDAGLowering::visitIntToPtr(User &I) { // What to do depends on the size of the integer and the size of the pointer. // We can either truncate, zero extend, or no-op, accordingly. - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType SrcVT = N.getValueType(); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); - if (MVT::getSizeInBits(DestVT) < MVT::getSizeInBits(SrcVT)) + SDValue N = getValue(I.getOperand(0)); + MVT SrcVT = N.getValueType(); + MVT DestVT = TLI.getValueType(I.getType()); + if (DestVT.bitsLT(SrcVT)) setValue(&I, DAG.getNode(ISD::TRUNCATE, DestVT, N)); else // Note: ZERO_EXTEND can handle cases where the sizes are equal too @@ -2335,8 +2659,8 @@ void SelectionDAGLowering::visitIntToPtr(User &I) { } void SelectionDAGLowering::visitBitCast(User &I) { - SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + SDValue N = getValue(I.getOperand(0)); + MVT DestVT = TLI.getValueType(I.getType()); // BitCast assures us that source and destination are the same size so this // is either a BIT_CONVERT or a no-op. @@ -2347,9 +2671,9 @@ void SelectionDAGLowering::visitBitCast(User &I) { } void SelectionDAGLowering::visitInsertElement(User &I) { - SDOperand InVec = getValue(I.getOperand(0)); - SDOperand InVal = getValue(I.getOperand(1)); - SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), + SDValue InVec = getValue(I.getOperand(0)); + SDValue InVal = getValue(I.getOperand(1)); + SDValue InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), getValue(I.getOperand(2))); setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, @@ -2358,26 +2682,92 @@ void SelectionDAGLowering::visitInsertElement(User &I) { } void SelectionDAGLowering::visitExtractElement(User &I) { - SDOperand InVec = getValue(I.getOperand(0)); - SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), + SDValue InVec = getValue(I.getOperand(0)); + SDValue InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), getValue(I.getOperand(1))); setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, TLI.getValueType(I.getType()), InVec, InIdx)); } void SelectionDAGLowering::visitShuffleVector(User &I) { - SDOperand V1 = getValue(I.getOperand(0)); - SDOperand V2 = getValue(I.getOperand(1)); - SDOperand Mask = getValue(I.getOperand(2)); + SDValue V1 = getValue(I.getOperand(0)); + SDValue V2 = getValue(I.getOperand(1)); + SDValue Mask = getValue(I.getOperand(2)); setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE, TLI.getValueType(I.getType()), V1, V2, Mask)); } +void SelectionDAGLowering::visitInsertValue(InsertValueInst &I) { + const Value *Op0 = I.getOperand(0); + const Value *Op1 = I.getOperand(1); + const Type *AggTy = I.getType(); + const Type *ValTy = Op1->getType(); + bool IntoUndef = isa(Op0); + bool FromUndef = isa(Op1); + + unsigned LinearIndex = ComputeLinearIndex(TLI, AggTy, + I.idx_begin(), I.idx_end()); + + SmallVector AggValueVTs; + ComputeValueVTs(TLI, AggTy, AggValueVTs); + SmallVector ValValueVTs; + ComputeValueVTs(TLI, ValTy, ValValueVTs); + + unsigned NumAggValues = AggValueVTs.size(); + unsigned NumValValues = ValValueVTs.size(); + SmallVector Values(NumAggValues); + + SDValue Agg = getValue(Op0); + SDValue Val = getValue(Op1); + unsigned i = 0; + // Copy the beginning value(s) from the original aggregate. + for (; i != LinearIndex; ++i) + Values[i] = IntoUndef ? DAG.getNode(ISD::UNDEF, AggValueVTs[i]) : + SDValue(Agg.Val, Agg.ResNo + i); + // Copy values from the inserted value(s). + for (; i != LinearIndex + NumValValues; ++i) + Values[i] = FromUndef ? DAG.getNode(ISD::UNDEF, AggValueVTs[i]) : + SDValue(Val.Val, Val.ResNo + i - LinearIndex); + // Copy remaining value(s) from the original aggregate. + for (; i != NumAggValues; ++i) + Values[i] = IntoUndef ? DAG.getNode(ISD::UNDEF, AggValueVTs[i]) : + SDValue(Agg.Val, Agg.ResNo + i); + + setValue(&I, DAG.getMergeValues(DAG.getVTList(&AggValueVTs[0], NumAggValues), + &Values[0], NumAggValues)); +} + +void SelectionDAGLowering::visitExtractValue(ExtractValueInst &I) { + const Value *Op0 = I.getOperand(0); + const Type *AggTy = Op0->getType(); + const Type *ValTy = I.getType(); + bool OutOfUndef = isa(Op0); + + unsigned LinearIndex = ComputeLinearIndex(TLI, AggTy, + I.idx_begin(), I.idx_end()); + + SmallVector ValValueVTs; + ComputeValueVTs(TLI, ValTy, ValValueVTs); + + unsigned NumValValues = ValValueVTs.size(); + SmallVector Values(NumValValues); + + SDValue Agg = getValue(Op0); + // Copy out the selected value(s). + for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i) + Values[i - LinearIndex] = + OutOfUndef ? DAG.getNode(ISD::UNDEF, Agg.Val->getValueType(Agg.ResNo + i)) : + SDValue(Agg.Val, Agg.ResNo + i); + + setValue(&I, DAG.getMergeValues(DAG.getVTList(&ValValueVTs[0], NumValValues), + &Values[0], NumValValues)); +} + void SelectionDAGLowering::visitGetElementPtr(User &I) { - SDOperand N = getValue(I.getOperand(0)); + SDValue N = getValue(I.getOperand(0)); const Type *Ty = I.getOperand(0)->getType(); for (GetElementPtrInst::op_iterator OI = I.op_begin()+1, E = I.op_end(); @@ -2407,13 +2797,13 @@ void SelectionDAGLowering::visitGetElementPtr(User &I) { // N = N + Idx * ElementSize; uint64_t ElementSize = TD->getABITypeSize(Ty); - SDOperand IdxN = getValue(Idx); + SDValue IdxN = getValue(Idx); // If the index is smaller or larger than intptr_t, truncate or extend // it. - if (IdxN.getValueType() < N.getValueType()) { + if (IdxN.getValueType().bitsLT(N.getValueType())) { IdxN = DAG.getNode(ISD::SIGN_EXTEND, N.getValueType(), IdxN); - } else if (IdxN.getValueType() > N.getValueType()) + } else if (IdxN.getValueType().bitsGT(N.getValueType())) IdxN = DAG.getNode(ISD::TRUNCATE, N.getValueType(), IdxN); // If this is a multiply by a power of two, turn it into a shl @@ -2426,7 +2816,7 @@ void SelectionDAGLowering::visitGetElementPtr(User &I) { continue; } - SDOperand Scale = DAG.getIntPtrConstant(ElementSize); + SDValue Scale = DAG.getIntPtrConstant(ElementSize); IdxN = DAG.getNode(ISD::MUL, N.getValueType(), IdxN, Scale); N = DAG.getNode(ISD::ADD, N.getValueType(), N, IdxN); } @@ -2446,11 +2836,11 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) { std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), I.getAlignment()); - SDOperand AllocSize = getValue(I.getArraySize()); - MVT::ValueType IntPtr = TLI.getPointerTy(); - if (IntPtr < AllocSize.getValueType()) + SDValue AllocSize = getValue(I.getArraySize()); + MVT IntPtr = TLI.getPointerTy(); + if (IntPtr.bitsLT(AllocSize.getValueType())) AllocSize = DAG.getNode(ISD::TRUNCATE, IntPtr, AllocSize); - else if (IntPtr > AllocSize.getValueType()) + else if (IntPtr.bitsGT(AllocSize.getValueType())) AllocSize = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, AllocSize); AllocSize = DAG.getNode(ISD::MUL, IntPtr, AllocSize, @@ -2472,10 +2862,10 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) { AllocSize = DAG.getNode(ISD::AND, AllocSize.getValueType(), AllocSize, DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1))); - SDOperand Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) }; - const MVT::ValueType *VTs = DAG.getNodeValueTypes(AllocSize.getValueType(), + SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) }; + const MVT *VTs = DAG.getNodeValueTypes(AllocSize.getValueType(), MVT::Other); - SDOperand DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, VTs, 2, Ops, 3); + SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, VTs, 2, Ops, 3); setValue(&I, DSA); DAG.setRoot(DSA.getValue(1)); @@ -2485,43 +2875,91 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) { } void SelectionDAGLowering::visitLoad(LoadInst &I) { - SDOperand Ptr = getValue(I.getOperand(0)); + const Value *SV = I.getOperand(0); + SDValue Ptr = getValue(SV); + + const Type *Ty = I.getType(); + bool isVolatile = I.isVolatile(); + unsigned Alignment = I.getAlignment(); + + SmallVector ValueVTs; + SmallVector Offsets; + ComputeValueVTs(TLI, Ty, ValueVTs, &Offsets); + unsigned NumValues = ValueVTs.size(); + if (NumValues == 0) + return; - SDOperand Root; + SDValue Root; + bool ConstantMemory = false; if (I.isVolatile()) + // Serialize volatile loads with other side effects. Root = getRoot(); - else { + else if (AA.pointsToConstantMemory(SV)) { + // Do not serialize (non-volatile) loads of constant memory with anything. + Root = DAG.getEntryNode(); + ConstantMemory = true; + } else { // Do not serialize non-volatile loads against each other. Root = DAG.getRoot(); } - setValue(&I, getLoadFrom(I.getType(), Ptr, I.getOperand(0), - Root, I.isVolatile(), I.getAlignment())); -} - -SDOperand SelectionDAGLowering::getLoadFrom(const Type *Ty, SDOperand Ptr, - const Value *SV, SDOperand Root, - bool isVolatile, - unsigned Alignment) { - SDOperand L = - DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, SV, 0, - isVolatile, Alignment); - - if (isVolatile) - DAG.setRoot(L.getValue(1)); - else - PendingLoads.push_back(L.getValue(1)); + SmallVector Values(NumValues); + SmallVector Chains(NumValues); + MVT PtrVT = Ptr.getValueType(); + for (unsigned i = 0; i != NumValues; ++i) { + SDValue L = DAG.getLoad(ValueVTs[i], Root, + DAG.getNode(ISD::ADD, PtrVT, Ptr, + DAG.getConstant(Offsets[i], PtrVT)), + SV, Offsets[i], + isVolatile, Alignment); + Values[i] = L; + Chains[i] = L.getValue(1); + } - return L; + if (!ConstantMemory) { + SDValue Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, + &Chains[0], NumValues); + if (isVolatile) + DAG.setRoot(Chain); + else + PendingLoads.push_back(Chain); + } + + setValue(&I, DAG.getMergeValues(DAG.getVTList(&ValueVTs[0], NumValues), + &Values[0], NumValues)); } void SelectionDAGLowering::visitStore(StoreInst &I) { Value *SrcV = I.getOperand(0); - SDOperand Src = getValue(SrcV); - SDOperand Ptr = getValue(I.getOperand(1)); - DAG.setRoot(DAG.getStore(getRoot(), Src, Ptr, I.getOperand(1), 0, - I.isVolatile(), I.getAlignment())); + Value *PtrV = I.getOperand(1); + + SmallVector ValueVTs; + SmallVector Offsets; + ComputeValueVTs(TLI, SrcV->getType(), ValueVTs, &Offsets); + unsigned NumValues = ValueVTs.size(); + if (NumValues == 0) + return; + + // Get the lowered operands. Note that we do this after + // checking if NumResults is zero, because with zero results + // the operands won't have values in the map. + SDValue Src = getValue(SrcV); + SDValue Ptr = getValue(PtrV); + + SDValue Root = getRoot(); + SmallVector Chains(NumValues); + MVT PtrVT = Ptr.getValueType(); + bool isVolatile = I.isVolatile(); + unsigned Alignment = I.getAlignment(); + for (unsigned i = 0; i != NumValues; ++i) + Chains[i] = DAG.getStore(Root, SDValue(Src.Val, Src.ResNo + i), + DAG.getNode(ISD::ADD, PtrVT, Ptr, + DAG.getConstant(Offsets[i], PtrVT)), + PtrV, Offsets[i], + isVolatile, Alignment); + + DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumValues)); } /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC @@ -2532,7 +2970,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, bool OnlyLoad = HasChain && I.onlyReadsMemory(); // Build the operand list. - SmallVector Ops; + SmallVector Ops; if (HasChain) { // If this intrinsic has side-effects, chainify it. if (OnlyLoad) { // We don't need to serialize loads against other loads. @@ -2547,20 +2985,20 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, // Add all operands of the call to the operand list. for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i) { - SDOperand Op = getValue(I.getOperand(i)); + SDValue Op = getValue(I.getOperand(i)); assert(TLI.isTypeLegal(Op.getValueType()) && "Intrinsic uses a non-legal type?"); Ops.push_back(Op); } - std::vector VTs; + std::vector VTs; if (I.getType() != Type::VoidTy) { - MVT::ValueType VT = TLI.getValueType(I.getType()); - if (MVT::isVector(VT)) { + MVT VT = TLI.getValueType(I.getType()); + if (VT.isVector()) { const VectorType *DestTy = cast(I.getType()); - MVT::ValueType EltVT = TLI.getValueType(DestTy->getElementType()); + MVT EltVT = TLI.getValueType(DestTy->getElementType()); - VT = MVT::getVectorType(EltVT, DestTy->getNumElements()); + VT = MVT::getVectorVT(EltVT, DestTy->getNumElements()); assert(VT != MVT::Other && "Intrinsic uses a non-legal type?"); } @@ -2570,10 +3008,10 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, if (HasChain) VTs.push_back(MVT::Other); - const MVT::ValueType *VTList = DAG.getNodeValueTypes(VTs); + const MVT *VTList = DAG.getNodeValueTypes(VTs); // Create the node. - SDOperand Result; + SDValue Result; if (!HasChain) Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VTList, VTs.size(), &Ops[0], Ops.size()); @@ -2585,7 +3023,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, &Ops[0], Ops.size()); if (HasChain) { - SDOperand Chain = Result.getValue(Result.Val->getNumValues()-1); + SDValue Chain = Result.getValue(Result.Val->getNumValues()-1); if (OnlyLoad) PendingLoads.push_back(Chain); else @@ -2593,7 +3031,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, } if (I.getType() != Type::VoidTy) { if (const VectorType *PTy = dyn_cast(I.getType())) { - MVT::ValueType VT = TLI.getValueType(PTy); + MVT VT = TLI.getValueType(PTy); Result = DAG.getNode(ISD::BIT_CONVERT, VT, Result); } setValue(&I, Result); @@ -2602,7 +3040,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, /// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V. static GlobalVariable *ExtractTypeInfo (Value *V) { - V = IntrinsicInst::StripPointerCasts(V); + V = V->stripPointerCasts(); GlobalVariable *GV = dyn_cast(V); assert ((GV || isa(V)) && "TypeInfo must be a global variable or NULL"); @@ -2663,6 +3101,22 @@ static void addCatchInfo(CallInst &I, MachineModuleInfo *MMI, } } + +/// Inlined utility function to implement binary input atomic intrinsics for +// visitIntrinsicCall: I is a call instruction +// Op is the associated NodeType for I +const char * +SelectionDAGLowering::implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op) { + SDValue Root = getRoot(); + SDValue L = DAG.getAtomic(Op, Root, + getValue(I.getOperand(1)), + getValue(I.getOperand(2)), + I.getOperand(1)); + setValue(&I, L); + DAG.setRoot(L.getValue(1)); + return 0; +} + /// visitIntrinsicCall - Lower the call to the specified intrinsic function. If /// we want to emit this as a call to a named external function, return the name /// otherwise lower it and return null. @@ -2691,36 +3145,58 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return "_longjmp"+!TLI.usesUnderscoreLongJmp(); break; case Intrinsic::memcpy_i32: - case Intrinsic::memcpy_i64: - visitMemIntrinsic(I, ISD::MEMCPY); + case Intrinsic::memcpy_i64: { + SDValue Op1 = getValue(I.getOperand(1)); + SDValue Op2 = getValue(I.getOperand(2)); + SDValue Op3 = getValue(I.getOperand(3)); + unsigned Align = cast(I.getOperand(4))->getZExtValue(); + DAG.setRoot(DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Align, false, + I.getOperand(1), 0, I.getOperand(2), 0)); return 0; + } case Intrinsic::memset_i32: - case Intrinsic::memset_i64: - visitMemIntrinsic(I, ISD::MEMSET); + case Intrinsic::memset_i64: { + SDValue Op1 = getValue(I.getOperand(1)); + SDValue Op2 = getValue(I.getOperand(2)); + SDValue Op3 = getValue(I.getOperand(3)); + unsigned Align = cast(I.getOperand(4))->getZExtValue(); + DAG.setRoot(DAG.getMemset(getRoot(), Op1, Op2, Op3, Align, + I.getOperand(1), 0)); return 0; + } case Intrinsic::memmove_i32: - case Intrinsic::memmove_i64: - visitMemIntrinsic(I, ISD::MEMMOVE); + case Intrinsic::memmove_i64: { + SDValue Op1 = getValue(I.getOperand(1)); + SDValue Op2 = getValue(I.getOperand(2)); + SDValue Op3 = getValue(I.getOperand(3)); + unsigned Align = cast(I.getOperand(4))->getZExtValue(); + + // If the source and destination are known to not be aliases, we can + // lower memmove as memcpy. + uint64_t Size = -1ULL; + if (ConstantSDNode *C = dyn_cast(Op3)) + Size = C->getValue(); + if (AA.alias(I.getOperand(1), Size, I.getOperand(2), Size) == + AliasAnalysis::NoAlias) { + DAG.setRoot(DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Align, false, + I.getOperand(1), 0, I.getOperand(2), 0)); + return 0; + } + + DAG.setRoot(DAG.getMemmove(getRoot(), Op1, Op2, Op3, Align, + I.getOperand(1), 0, I.getOperand(2), 0)); return 0; - + } case Intrinsic::dbg_stoppoint: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); DbgStopPointInst &SPI = cast(I); if (MMI && SPI.getContext() && MMI->Verify(SPI.getContext())) { - SDOperand Ops[5]; - - Ops[0] = getRoot(); - Ops[1] = getValue(SPI.getLineValue()); - Ops[2] = getValue(SPI.getColumnValue()); - DebugInfoDesc *DD = MMI->getDescFor(SPI.getContext()); assert(DD && "Not a debug information descriptor"); - CompileUnitDesc *CompileUnit = cast(DD); - - Ops[3] = DAG.getString(CompileUnit->getFileName()); - Ops[4] = DAG.getString(CompileUnit->getDirectory()); - - DAG.setRoot(DAG.getNode(ISD::LOCATION, MVT::Other, Ops, 5)); + DAG.setRoot(DAG.getDbgStopPoint(getRoot(), + SPI.getLine(), + SPI.getColumn(), + cast(DD))); } return 0; @@ -2730,9 +3206,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DbgRegionStartInst &RSI = cast(I); if (MMI && RSI.getContext() && MMI->Verify(RSI.getContext())) { unsigned LabelID = MMI->RecordRegionStart(RSI.getContext()); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), - DAG.getConstant(LabelID, MVT::i32), - DAG.getConstant(0, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::DBG_LABEL, getRoot(), LabelID)); } return 0; @@ -2742,9 +3216,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DbgRegionEndInst &REI = cast(I); if (MMI && REI.getContext() && MMI->Verify(REI.getContext())) { unsigned LabelID = MMI->RecordRegionEnd(REI.getContext()); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), - DAG.getConstant(LabelID, MVT::i32), - DAG.getConstant(0, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::DBG_LABEL, getRoot(), LabelID)); } return 0; @@ -2761,8 +3233,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { assert(DD && "Not a debug information descriptor"); SubprogramDesc *Subprogram = cast(DD); const CompileUnitDesc *CompileUnit = Subprogram->getFile(); - unsigned SrcFile = MMI->RecordSource(CompileUnit->getDirectory(), - CompileUnit->getFileName()); + unsigned SrcFile = MMI->RecordSource(CompileUnit); // Record the source line but does create a label. It will be emitted // at asm emission time. MMI->RecordSourceLine(Subprogram->getLine(), 0, SrcFile); @@ -2781,32 +3252,28 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::eh_exception: { - if (ExceptionHandling) { - if (!CurMBB->isLandingPad()) { - // FIXME: Mark exception register as live in. Hack for PR1508. - unsigned Reg = TLI.getExceptionAddressRegister(); - if (Reg) CurMBB->addLiveIn(Reg); - } - // Insert the EXCEPTIONADDR instruction. - SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other); - SDOperand Ops[1]; - Ops[0] = DAG.getRoot(); - SDOperand Op = DAG.getNode(ISD::EXCEPTIONADDR, VTs, Ops, 1); - setValue(&I, Op); - DAG.setRoot(Op.getValue(1)); - } else { - setValue(&I, DAG.getConstant(0, TLI.getPointerTy())); + if (!CurMBB->isLandingPad()) { + // FIXME: Mark exception register as live in. Hack for PR1508. + unsigned Reg = TLI.getExceptionAddressRegister(); + if (Reg) CurMBB->addLiveIn(Reg); } + // Insert the EXCEPTIONADDR instruction. + SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other); + SDValue Ops[1]; + Ops[0] = DAG.getRoot(); + SDValue Op = DAG.getNode(ISD::EXCEPTIONADDR, VTs, Ops, 1); + setValue(&I, Op); + DAG.setRoot(Op.getValue(1)); return 0; } case Intrinsic::eh_selector_i32: case Intrinsic::eh_selector_i64: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - MVT::ValueType VT = (Intrinsic == Intrinsic::eh_selector_i32 ? + MVT VT = (Intrinsic == Intrinsic::eh_selector_i32 ? MVT::i32 : MVT::i64); - if (ExceptionHandling && MMI) { + if (MMI) { if (CurMBB->isLandingPad()) addCatchInfo(I, MMI, CurMBB); else { @@ -2820,10 +3287,10 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { // Insert the EHSELECTION instruction. SDVTList VTs = DAG.getVTList(VT, MVT::Other); - SDOperand Ops[2]; + SDValue Ops[2]; Ops[0] = getValue(I.getOperand(1)); Ops[1] = getRoot(); - SDOperand Op = DAG.getNode(ISD::EHSELECTION, VTs, Ops, 2); + SDValue Op = DAG.getNode(ISD::EHSELECTION, VTs, Ops, 2); setValue(&I, Op); DAG.setRoot(Op.getValue(1)); } else { @@ -2836,7 +3303,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::eh_typeid_for_i32: case Intrinsic::eh_typeid_for_i64: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - MVT::ValueType VT = (Intrinsic == Intrinsic::eh_typeid_for_i32 ? + MVT VT = (Intrinsic == Intrinsic::eh_typeid_for_i32 ? MVT::i32 : MVT::i64); if (MMI) { @@ -2856,11 +3323,11 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::eh_return: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - if (MMI && ExceptionHandling) { + if (MMI) { MMI->setCallsEHReturn(true); DAG.setRoot(DAG.getNode(ISD::EH_RETURN, MVT::Other, - getRoot(), + getControlRoot(), getValue(I.getOperand(1)), getValue(I.getOperand(2)))); } else { @@ -2879,32 +3346,27 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::eh_dwarf_cfa: { - if (ExceptionHandling) { - MVT::ValueType VT = getValue(I.getOperand(1)).getValueType(); - SDOperand CfaArg; - if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy())) - CfaArg = DAG.getNode(ISD::TRUNCATE, - TLI.getPointerTy(), getValue(I.getOperand(1))); - else - CfaArg = DAG.getNode(ISD::SIGN_EXTEND, - TLI.getPointerTy(), getValue(I.getOperand(1))); - - SDOperand Offset = DAG.getNode(ISD::ADD, - TLI.getPointerTy(), - DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, - TLI.getPointerTy()), - CfaArg); - setValue(&I, DAG.getNode(ISD::ADD, - TLI.getPointerTy(), - DAG.getNode(ISD::FRAMEADDR, - TLI.getPointerTy(), - DAG.getConstant(0, - TLI.getPointerTy())), - Offset)); - } else { - setValue(&I, DAG.getConstant(0, TLI.getPointerTy())); - } - + MVT VT = getValue(I.getOperand(1)).getValueType(); + SDValue CfaArg; + if (VT.bitsGT(TLI.getPointerTy())) + CfaArg = DAG.getNode(ISD::TRUNCATE, + TLI.getPointerTy(), getValue(I.getOperand(1))); + else + CfaArg = DAG.getNode(ISD::SIGN_EXTEND, + TLI.getPointerTy(), getValue(I.getOperand(1))); + + SDValue Offset = DAG.getNode(ISD::ADD, + TLI.getPointerTy(), + DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, + TLI.getPointerTy()), + CfaArg); + setValue(&I, DAG.getNode(ISD::ADD, + TLI.getPointerTy(), + DAG.getNode(ISD::FRAMEADDR, + TLI.getPointerTy(), + DAG.getConstant(0, + TLI.getPointerTy())), + Offset)); return 0; } @@ -2936,13 +3398,13 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { getValue(I.getOperand(2)))); return 0; case Intrinsic::pcmarker: { - SDOperand Tmp = getValue(I.getOperand(1)); + SDValue Tmp = getValue(I.getOperand(1)); DAG.setRoot(DAG.getNode(ISD::PCMARKER, MVT::Other, getRoot(), Tmp)); return 0; } case Intrinsic::readcyclecounter: { - SDOperand Op = getRoot(); - SDOperand Tmp = DAG.getNode(ISD::READCYCLECOUNTER, + SDValue Op = getRoot(); + SDValue Tmp = DAG.getNode(ISD::READCYCLECOUNTER, DAG.getNodeValueTypes(MVT::i64, MVT::Other), 2, &Op, 1); setValue(&I, Tmp); @@ -2965,36 +3427,36 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { getValue(I.getOperand(1)))); return 0; case Intrinsic::cttz: { - SDOperand Arg = getValue(I.getOperand(1)); - MVT::ValueType Ty = Arg.getValueType(); - SDOperand result = DAG.getNode(ISD::CTTZ, Ty, Arg); + SDValue Arg = getValue(I.getOperand(1)); + MVT Ty = Arg.getValueType(); + SDValue result = DAG.getNode(ISD::CTTZ, Ty, Arg); setValue(&I, result); return 0; } case Intrinsic::ctlz: { - SDOperand Arg = getValue(I.getOperand(1)); - MVT::ValueType Ty = Arg.getValueType(); - SDOperand result = DAG.getNode(ISD::CTLZ, Ty, Arg); + SDValue Arg = getValue(I.getOperand(1)); + MVT Ty = Arg.getValueType(); + SDValue result = DAG.getNode(ISD::CTLZ, Ty, Arg); setValue(&I, result); return 0; } case Intrinsic::ctpop: { - SDOperand Arg = getValue(I.getOperand(1)); - MVT::ValueType Ty = Arg.getValueType(); - SDOperand result = DAG.getNode(ISD::CTPOP, Ty, Arg); + SDValue Arg = getValue(I.getOperand(1)); + MVT Ty = Arg.getValueType(); + SDValue result = DAG.getNode(ISD::CTPOP, Ty, Arg); setValue(&I, result); return 0; } case Intrinsic::stacksave: { - SDOperand Op = getRoot(); - SDOperand Tmp = DAG.getNode(ISD::STACKSAVE, + SDValue Op = getRoot(); + SDValue Tmp = DAG.getNode(ISD::STACKSAVE, DAG.getNodeValueTypes(TLI.getPointerTy(), MVT::Other), 2, &Op, 1); setValue(&I, Tmp); DAG.setRoot(Tmp.getValue(1)); return 0; } case Intrinsic::stackrestore: { - SDOperand Tmp = getValue(I.getOperand(1)); + SDValue Tmp = getValue(I.getOperand(1)); DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, MVT::Other, getRoot(), Tmp)); return 0; } @@ -3003,10 +3465,9 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return 0; case Intrinsic::init_trampoline: { - const Function *F = - cast(IntrinsicInst::StripPointerCasts(I.getOperand(2))); + const Function *F = cast(I.getOperand(2)->stripPointerCasts()); - SDOperand Ops[6]; + SDValue Ops[6]; Ops[0] = getRoot(); Ops[1] = getValue(I.getOperand(1)); Ops[2] = getValue(I.getOperand(2)); @@ -3014,7 +3475,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { Ops[4] = DAG.getSrcValue(I.getOperand(1)); Ops[5] = DAG.getSrcValue(F); - SDOperand Tmp = DAG.getNode(ISD::TRAMPOLINE, + SDValue Tmp = DAG.getNode(ISD::TRAMPOLINE, DAG.getNodeValueTypes(TLI.getPointerTy(), MVT::Other), 2, Ops, 6); @@ -3049,7 +3510,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return 0; } case Intrinsic::prefetch: { - SDOperand Ops[4]; + SDValue Ops[4]; Ops[0] = getRoot(); Ops[1] = getValue(I.getOperand(1)); Ops[2] = getValue(I.getOperand(2)); @@ -3059,7 +3520,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::memory_barrier: { - SDOperand Ops[6]; + SDValue Ops[6]; Ops[0] = getRoot(); for (int x = 1; x < 6; ++x) Ops[x] = getValue(I.getOperand(x)); @@ -3067,43 +3528,44 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DAG.setRoot(DAG.getNode(ISD::MEMBARRIER, MVT::Other, &Ops[0], 6)); return 0; } - case Intrinsic::atomic_lcs: { - SDOperand Root = getRoot(); - SDOperand O3 = getValue(I.getOperand(3)); - SDOperand L = DAG.getAtomic(ISD::ATOMIC_LCS, Root, + case Intrinsic::atomic_cmp_swap: { + SDValue Root = getRoot(); + SDValue L = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, Root, getValue(I.getOperand(1)), getValue(I.getOperand(2)), - O3, O3.getValueType()); - setValue(&I, L); - DAG.setRoot(L.getValue(1)); - return 0; - } - case Intrinsic::atomic_las: { - SDOperand Root = getRoot(); - SDOperand O2 = getValue(I.getOperand(2)); - SDOperand L = DAG.getAtomic(ISD::ATOMIC_LAS, Root, - getValue(I.getOperand(1)), - O2, O2.getValueType()); - setValue(&I, L); - DAG.setRoot(L.getValue(1)); - return 0; - } - case Intrinsic::atomic_swap: { - SDOperand Root = getRoot(); - SDOperand O2 = getValue(I.getOperand(2)); - SDOperand L = DAG.getAtomic(ISD::ATOMIC_SWAP, Root, - getValue(I.getOperand(1)), - O2, O2.getValueType()); + getValue(I.getOperand(3)), + I.getOperand(1)); setValue(&I, L); DAG.setRoot(L.getValue(1)); return 0; } - + case Intrinsic::atomic_load_add: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_ADD); + case Intrinsic::atomic_load_sub: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_SUB); + case Intrinsic::atomic_load_and: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_AND); + case Intrinsic::atomic_load_or: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_OR); + case Intrinsic::atomic_load_xor: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_XOR); + case Intrinsic::atomic_load_nand: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_NAND); + case Intrinsic::atomic_load_min: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MIN); + case Intrinsic::atomic_load_max: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MAX); + case Intrinsic::atomic_load_umin: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMIN); + case Intrinsic::atomic_load_umax: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMAX); + case Intrinsic::atomic_swap: + return implVisitBinaryAtomic(I, ISD::ATOMIC_SWAP); } } -void SelectionDAGLowering::LowerCallTo(CallSite CS, SDOperand Callee, +void SelectionDAGLowering::LowerCallTo(CallSite CS, SDValue Callee, bool IsTailCall, MachineBasicBlock *LandingPad) { const PointerType *PT = cast(CS.getCalledValue()->getType()); @@ -3116,7 +3578,7 @@ void SelectionDAGLowering::LowerCallTo(CallSite CS, SDOperand Callee, Args.reserve(CS.arg_size()); for (CallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); i != e; ++i) { - SDOperand ArgNode = getValue(*i); + SDValue ArgNode = getValue(*i); Entry.Node = ArgNode; Entry.Ty = (*i)->getType(); unsigned attrInd = i - CS.arg_begin() + 1; @@ -3130,20 +3592,17 @@ void SelectionDAGLowering::LowerCallTo(CallSite CS, SDOperand Callee, Args.push_back(Entry); } - bool MarkTryRange = LandingPad || - // C++ requires special handling of 'nounwind' calls. - (CS.doesNotThrow()); - - if (MarkTryRange && ExceptionHandling && MMI) { + if (LandingPad && MMI) { // Insert a label before the invoke call to mark the try range. This can be // used to detect deletion of the invoke via the MachineModuleInfo. BeginLabel = MMI->NextLabelID(); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), - DAG.getConstant(BeginLabel, MVT::i32), - DAG.getConstant(1, MVT::i32))); + // Both PendingLoads and PendingExports must be flushed here; + // this call might not return. + (void)getRoot(); + DAG.setRoot(DAG.getLabel(ISD::EH_LABEL, getControlRoot(), BeginLabel)); } - std::pair Result = + std::pair Result = TLI.LowerCallTo(getRoot(), CS.getType(), CS.paramHasAttr(0, ParamAttr::SExt), CS.paramHasAttr(0, ParamAttr::ZExt), @@ -3153,13 +3612,11 @@ void SelectionDAGLowering::LowerCallTo(CallSite CS, SDOperand Callee, setValue(CS.getInstruction(), Result.first); DAG.setRoot(Result.second); - if (MarkTryRange && ExceptionHandling && MMI) { + if (LandingPad && MMI) { // Insert a label at the end of the invoke call to mark the try range. This // can be used to detect deletion of the invoke via the MachineModuleInfo. EndLabel = MMI->NextLabelID(); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), - DAG.getConstant(EndLabel, MVT::i32), - DAG.getConstant(1, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::EH_LABEL, getRoot(), EndLabel)); // Inform MachineModuleInfo of range. MMI->addInvoke(LandingPad, BeginLabel, EndLabel); @@ -3190,8 +3647,8 @@ void SelectionDAGLowering::visitCall(CallInst &I) { I.getOperand(1)->getType()->isFloatingPoint() && I.getType() == I.getOperand(1)->getType() && I.getType() == I.getOperand(2)->getType()) { - SDOperand LHS = getValue(I.getOperand(1)); - SDOperand RHS = getValue(I.getOperand(2)); + SDValue LHS = getValue(I.getOperand(1)); + SDValue RHS = getValue(I.getOperand(2)); setValue(&I, DAG.getNode(ISD::FCOPYSIGN, LHS.getValueType(), LHS, RHS)); return; @@ -3203,7 +3660,7 @@ void SelectionDAGLowering::visitCall(CallInst &I) { if (I.getNumOperands() == 2 && // Basic sanity checks. I.getOperand(1)->getType()->isFloatingPoint() && I.getType() == I.getOperand(1)->getType()) { - SDOperand Tmp = getValue(I.getOperand(1)); + SDValue Tmp = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FABS, Tmp.getValueType(), Tmp)); return; } @@ -3214,7 +3671,7 @@ void SelectionDAGLowering::visitCall(CallInst &I) { if (I.getNumOperands() == 2 && // Basic sanity checks. I.getOperand(1)->getType()->isFloatingPoint() && I.getType() == I.getOperand(1)->getType()) { - SDOperand Tmp = getValue(I.getOperand(1)); + SDValue Tmp = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FSIN, Tmp.getValueType(), Tmp)); return; } @@ -3225,7 +3682,7 @@ void SelectionDAGLowering::visitCall(CallInst &I) { if (I.getNumOperands() == 2 && // Basic sanity checks. I.getOperand(1)->getType()->isFloatingPoint() && I.getType() == I.getOperand(1)->getType()) { - SDOperand Tmp = getValue(I.getOperand(1)); + SDValue Tmp = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FCOS, Tmp.getValueType(), Tmp)); return; } @@ -3236,7 +3693,7 @@ void SelectionDAGLowering::visitCall(CallInst &I) { return; } - SDOperand Callee; + SDValue Callee; if (!RenameFn) Callee = getValue(I.getOperand(0)); else @@ -3250,57 +3707,144 @@ void SelectionDAGLowering::visitCall(CallInst &I) { /// this value and returns the result as a ValueVT value. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. -SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG, - SDOperand &Chain, SDOperand *Flag)const{ - // Copy the legal parts from the registers. - unsigned NumParts = Regs.size(); - SmallVector Parts(NumParts); - for (unsigned i = 0; i != NumParts; ++i) { - SDOperand Part = Flag ? - DAG.getCopyFromReg(Chain, Regs[i], RegVT, *Flag) : - DAG.getCopyFromReg(Chain, Regs[i], RegVT); - Chain = Part.getValue(1); - if (Flag) - *Flag = Part.getValue(2); - Parts[i] = Part; - } +SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, + SDValue &Chain, + SDValue *Flag) const { + // Assemble the legal parts into the final values. + SmallVector Values(ValueVTs.size()); + SmallVector Parts; + for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { + // Copy the legal parts from the registers. + MVT ValueVT = ValueVTs[Value]; + unsigned NumRegs = TLI->getNumRegisters(ValueVT); + MVT RegisterVT = RegVTs[Value]; + + Parts.resize(NumRegs); + for (unsigned i = 0; i != NumRegs; ++i) { + SDValue P; + if (Flag == 0) + P = DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT); + else { + P = DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT, *Flag); + *Flag = P.getValue(2); + } + Chain = P.getValue(1); + + // If the source register was virtual and if we know something about it, + // add an assert node. + if (TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) && + RegisterVT.isInteger() && !RegisterVT.isVector()) { + unsigned SlotNo = Regs[Part+i]-TargetRegisterInfo::FirstVirtualRegister; + FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); + if (FLI.LiveOutRegInfo.size() > SlotNo) { + FunctionLoweringInfo::LiveOutInfo &LOI = FLI.LiveOutRegInfo[SlotNo]; + + unsigned RegSize = RegisterVT.getSizeInBits(); + unsigned NumSignBits = LOI.NumSignBits; + unsigned NumZeroBits = LOI.KnownZero.countLeadingOnes(); + + // FIXME: We capture more information than the dag can represent. For + // now, just use the tightest assertzext/assertsext possible. + bool isSExt = true; + MVT FromVT(MVT::Other); + if (NumSignBits == RegSize) + isSExt = true, FromVT = MVT::i1; // ASSERT SEXT 1 + else if (NumZeroBits >= RegSize-1) + isSExt = false, FromVT = MVT::i1; // ASSERT ZEXT 1 + else if (NumSignBits > RegSize-8) + isSExt = true, FromVT = MVT::i8; // ASSERT SEXT 8 + else if (NumZeroBits >= RegSize-9) + isSExt = false, FromVT = MVT::i8; // ASSERT ZEXT 8 + else if (NumSignBits > RegSize-16) + isSExt = true, FromVT = MVT::i16; // ASSERT SEXT 16 + else if (NumZeroBits >= RegSize-17) + isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16 + else if (NumSignBits > RegSize-32) + isSExt = true, FromVT = MVT::i32; // ASSERT SEXT 32 + else if (NumZeroBits >= RegSize-33) + isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32 + + if (FromVT != MVT::Other) { + P = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, + RegisterVT, P, DAG.getValueType(FromVT)); + + } + } + } + + Parts[Part+i] = P; + } - // Assemble the legal parts into the final value. - return getCopyFromParts(DAG, &Parts[0], NumParts, RegVT, ValueVT, - ISD::DELETED_NODE); + Values[Value] = getCopyFromParts(DAG, &Parts[Part], NumRegs, RegisterVT, + ValueVT); + Part += NumRegs; + } + + return DAG.getMergeValues(DAG.getVTList(&ValueVTs[0], ValueVTs.size()), + &Values[0], ValueVTs.size()); } /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the /// specified value into the registers specified by this object. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. -void RegsForValue::getCopyToRegs(SDOperand Val, SelectionDAG &DAG, - SDOperand &Chain, SDOperand *Flag) const { +void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, + SDValue &Chain, SDValue *Flag) const { // Get the list of the values's legal parts. - unsigned NumParts = Regs.size(); - SmallVector Parts(NumParts); - getCopyToParts(DAG, Val, &Parts[0], NumParts, RegVT); + unsigned NumRegs = Regs.size(); + SmallVector Parts(NumRegs); + for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { + MVT ValueVT = ValueVTs[Value]; + unsigned NumParts = TLI->getNumRegisters(ValueVT); + MVT RegisterVT = RegVTs[Value]; + + getCopyToParts(DAG, Val.getValue(Val.ResNo + Value), + &Parts[Part], NumParts, RegisterVT); + Part += NumParts; + } // Copy the parts into the registers. - for (unsigned i = 0; i != NumParts; ++i) { - SDOperand Part = Flag ? - DAG.getCopyToReg(Chain, Regs[i], Parts[i], *Flag) : - DAG.getCopyToReg(Chain, Regs[i], Parts[i]); - Chain = Part.getValue(0); - if (Flag) + SmallVector Chains(NumRegs); + for (unsigned i = 0; i != NumRegs; ++i) { + SDValue Part; + if (Flag == 0) + Part = DAG.getCopyToReg(Chain, Regs[i], Parts[i]); + else { + Part = DAG.getCopyToReg(Chain, Regs[i], Parts[i], *Flag); *Flag = Part.getValue(1); + } + Chains[i] = Part.getValue(0); } + + if (NumRegs == 1 || Flag) + // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is + // flagged to it. That is the CopyToReg nodes and the user are considered + // a single scheduling unit. If we create a TokenFactor and return it as + // chain, then the TokenFactor is both a predecessor (operand) of the + // user as well as a successor (the TF operands are flagged to the user). + // c1, f1 = CopyToReg + // c2, f2 = CopyToReg + // c3 = TokenFactor c1, c2 + // ... + // = op c3, ..., f2 + Chain = Chains[NumRegs-1]; + else + Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumRegs); } /// AddInlineAsmOperands - Add this value to the specified inlineasm node /// operand list. This adds the code marker and includes the number of /// values added into it. void RegsForValue::AddInlineAsmOperands(unsigned Code, SelectionDAG &DAG, - std::vector &Ops) const { - MVT::ValueType IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); + std::vector &Ops) const { + MVT IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); Ops.push_back(DAG.getTargetConstant(Code | (Regs.size() << 3), IntPtrTy)); - for (unsigned i = 0, e = Regs.size(); i != e; ++i) - Ops.push_back(DAG.getRegister(Regs[i], RegVT)); + for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { + unsigned NumRegs = TLI->getNumRegisters(ValueVTs[Value]); + MVT RegisterVT = RegVTs[Value]; + for (unsigned i = 0; i != NumRegs; ++i) + Ops.push_back(DAG.getRegister(Regs[Reg++], RegisterVT)); + } } /// isAllocatableRegister - If the specified register is safe to allocate, @@ -3310,11 +3854,11 @@ static const TargetRegisterClass * isAllocatableRegister(unsigned Reg, MachineFunction &MF, const TargetLowering &TLI, const TargetRegisterInfo *TRI) { - MVT::ValueType FoundVT = MVT::Other; + MVT FoundVT = MVT::Other; const TargetRegisterClass *FoundRC = 0; for (TargetRegisterInfo::regclass_iterator RCI = TRI->regclass_begin(), E = TRI->regclass_end(); RCI != E; ++RCI) { - MVT::ValueType ThisVT = MVT::Other; + MVT ThisVT = MVT::Other; const TargetRegisterClass *RC = *RCI; // If none of the the value types for this register class are valid, we @@ -3325,8 +3869,7 @@ isAllocatableRegister(unsigned Reg, MachineFunction &MF, // If we have already found this register in a different register class, // choose the one with the largest VT specified. For example, on // PowerPC, we favor f64 register classes over f32. - if (FoundVT == MVT::Other || - MVT::getSizeInBits(FoundVT) < MVT::getSizeInBits(*I)) { + if (FoundVT == MVT::Other || FoundVT.bitsLT(*I)) { ThisVT = *I; break; } @@ -3360,13 +3903,13 @@ struct SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { /// CallOperand - If this is the result output operand or a clobber /// this is null, otherwise it is the incoming operand to the CallInst. /// This gets modified as the asm is processed. - SDOperand CallOperand; + SDValue CallOperand; /// AssignedRegs - If this is a register or register class operand, this /// contains the set of register corresponding to the operand. RegsForValue AssignedRegs; - SDISelAsmOperandInfo(const InlineAsm::ConstraintInfo &info) + explicit SDISelAsmOperandInfo(const InlineAsm::ConstraintInfo &info) : TargetLowering::AsmOperandInfo(info), CallOperand(0,0) { } @@ -3441,7 +3984,7 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, bool HasEarlyClobber, MachineFunction &MF = DAG.getMachineFunction(); - std::vector Regs; + SmallVector Regs; // If this is a constraint for a single physreg, or a constraint for a // register class, find it. @@ -3452,8 +3995,8 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, bool HasEarlyClobber, unsigned NumRegs = 1; if (OpInfo.ConstraintVT != MVT::Other) NumRegs = TLI.getNumRegisters(OpInfo.ConstraintVT); - MVT::ValueType RegVT; - MVT::ValueType ValueVT = OpInfo.ConstraintVT; + MVT RegVT; + MVT ValueVT = OpInfo.ConstraintVT; // If this is a constraint for a specific physical register, like {r17}, @@ -3473,18 +4016,17 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, bool HasEarlyClobber, // If this is an expanded reference, add the rest of the regs to Regs. if (NumRegs != 1) { TargetRegisterClass::iterator I = PhysReg.second->begin(); - TargetRegisterClass::iterator E = PhysReg.second->end(); for (; *I != PhysReg.first; ++I) - assert(I != E && "Didn't find reg!"); + assert(I != PhysReg.second->end() && "Didn't find reg!"); // Already added the first reg. --NumRegs; ++I; for (; NumRegs; --NumRegs, ++I) { - assert(I != E && "Ran out of registers to allocate!"); + assert(I != PhysReg.second->end() && "Ran out of registers to allocate!"); Regs.push_back(*I); } } - OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); + OpInfo.AssignedRegs = RegsForValue(TLI, Regs, RegVT, ValueVT); const TargetRegisterInfo *TRI = DAG.getTarget().getRegisterInfo(); OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs, *TRI); return; @@ -3513,7 +4055,7 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, bool HasEarlyClobber, for (; NumRegs; --NumRegs) Regs.push_back(RegInfo.createVirtualRegister(PhysReg.second)); - OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); + OpInfo.AssignedRegs = RegsForValue(TLI, Regs, RegVT, ValueVT); return; } @@ -3563,7 +4105,7 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, bool HasEarlyClobber, for (unsigned i = RegStart; i != RegEnd; ++i) Regs.push_back(RegClassRegs[i]); - OpInfo.AssignedRegs = RegsForValue(Regs, *RC->vt_begin(), + OpInfo.AssignedRegs = RegsForValue(TLI, Regs, *RC->vt_begin(), OpInfo.ConstraintVT); OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs, *TRI); return; @@ -3571,7 +4113,6 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, bool HasEarlyClobber, } // Otherwise, we couldn't allocate enough registers for this. - return; } @@ -3583,8 +4124,8 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { /// ConstraintOperands - Information about all of the constraints. std::vector ConstraintOperands; - SDOperand Chain = getRoot(); - SDOperand Flag; + SDValue Chain = getRoot(); + SDValue Flag; std::set OutputRegs, InputRegs; @@ -3599,23 +4140,31 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { bool SawEarlyClobber = false; unsigned ArgNo = 0; // ArgNo - The argument of the CallInst. + unsigned ResNo = 0; // ResNo - The result number of the next output. for (unsigned i = 0, e = ConstraintInfos.size(); i != e; ++i) { ConstraintOperands.push_back(SDISelAsmOperandInfo(ConstraintInfos[i])); SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back(); - MVT::ValueType OpVT = MVT::Other; + MVT OpVT = MVT::Other; // Compute the value type for each operand. switch (OpInfo.Type) { case InlineAsm::isOutput: - if (!OpInfo.isIndirect) { - // The return value of the call is this value. As such, there is no - // corresponding argument. - assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); - OpVT = TLI.getValueType(CS.getType()); - } else { + // Indirect outputs just consume an argument. + if (OpInfo.isIndirect) { OpInfo.CallOperandVal = CS.getArgument(ArgNo++); + break; + } + // The return value of the call is this value. As such, there is no + // corresponding argument. + assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); + if (const StructType *STy = dyn_cast(CS.getType())) { + OpVT = TLI.getValueType(STy->getElementType(ResNo)); + } else { + assert(ResNo == 0 && "Asm only has one result!"); + OpVT = TLI.getValueType(CS.getType()); } + ++ResNo; break; case InlineAsm::isInput: OpInfo.CallOperandVal = CS.getArgument(ArgNo++); @@ -3628,10 +4177,8 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // If this is an input or an indirect output, process the call argument. // BasicBlocks are labels, currently appearing only in asm's. if (OpInfo.CallOperandVal) { - if (isa(OpInfo.CallOperandVal)) - OpInfo.CallOperand = - DAG.getBasicBlock(FuncInfo.MBBMap[cast( - OpInfo.CallOperandVal)]); + if (BasicBlock *BB = dyn_cast(OpInfo.CallOperandVal)) + OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]); else { OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); const Type *OpTy = OpInfo.CallOperandVal->getType(); @@ -3640,9 +4187,9 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { if (OpInfo.isIndirect) OpTy = cast(OpTy)->getElementType(); - // If OpTy is not a first-class value, it may be a struct/union that we + // If OpTy is not a single value, it may be a struct/union that we // can tile with integers. - if (!OpTy->isFirstClassType() && OpTy->isSized()) { + if (!OpTy->isSingleValueType() && OpTy->isSized()) { unsigned BitSize = TD->getTypeSizeInBits(OpTy); switch (BitSize) { default: break; @@ -3663,7 +4210,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { OpInfo.ConstraintVT = OpVT; // Compute the constraint code and ConstraintType to use. - OpInfo.ComputeConstraintToUse(TLI); + TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); // Keep track of whether we see an earlyclobber. SawEarlyClobber |= OpInfo.isEarlyClobber; @@ -3709,7 +4256,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { unsigned Align = TLI.getTargetData()->getPrefTypeAlignment(Ty); MachineFunction &MF = DAG.getMachineFunction(); int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align); - SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy()); + SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy()); Chain = DAG.getStore(Chain, OpInfo.CallOperand, StackSlot, NULL, 0); OpInfo.CallOperand = StackSlot; } @@ -3740,8 +4287,8 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { } // AsmNodeOperands - The operands for the ISD::INLINEASM node. - std::vector AsmNodeOperands; - AsmNodeOperands.push_back(SDOperand()); // reserve space for input chain + std::vector AsmNodeOperands; + AsmNodeOperands.push_back(SDValue()); // reserve space for input chain AsmNodeOperands.push_back( DAG.getTargetExternalSymbol(IA->getAsmString().c_str(), MVT::Other)); @@ -3749,7 +4296,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // Loop over all of the inputs, copying the operand values into the // appropriate registers and processing the output regs. RegsForValue RetValRegs; - + // IndirectStoresToEmit - The set of stores to emit after the inline asm node. std::vector > IndirectStoresToEmit; @@ -3776,20 +4323,21 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // Copy the output from the appropriate register. Find a register that // we can use. if (OpInfo.AssignedRegs.Regs.empty()) { - cerr << "Couldn't allocate output reg for contraint '" + cerr << "Couldn't allocate output reg for constraint '" << OpInfo.ConstraintCode << "'!\n"; exit(1); } - if (!OpInfo.isIndirect) { - // This is the result value of the call. - assert(RetValRegs.Regs.empty() && - "Cannot have multiple output constraints yet!"); - assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); - RetValRegs = OpInfo.AssignedRegs; - } else { + // If this is an indirect operand, store through the pointer after the + // asm. + if (OpInfo.isIndirect) { IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs, OpInfo.CallOperandVal)); + } else { + // This is the result value of the call. + assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); + // Concatenate this output onto the outputs list. + RetValRegs.append(OpInfo.AssignedRegs); } // Add information to the INLINEASM node to know that this register is @@ -3799,7 +4347,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { break; } case InlineAsm::isInput: { - SDOperand InOperandVal = OpInfo.CallOperand; + SDValue InOperandVal = OpInfo.CallOperand; if (isdigit(OpInfo.ConstraintCode[0])) { // Matching constraint? // If this is required to match an output register we have already set, @@ -3824,8 +4372,9 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { if ((NumOps & 7) == 2 /*REGDEF*/) { // Add NumOps>>3 registers to MatchedRegs. RegsForValue MatchedRegs; - MatchedRegs.ValueVT = InOperandVal.getValueType(); - MatchedRegs.RegVT = AsmNodeOperands[CurOp+1].getValueType(); + MatchedRegs.TLI = &TLI; + MatchedRegs.ValueVTs.push_back(InOperandVal.getValueType()); + MatchedRegs.RegVTs.push_back(AsmNodeOperands[CurOp+1].getValueType()); for (unsigned i = 0, e = NumOps>>3; i != e; ++i) { unsigned Reg = cast(AsmNodeOperands[++CurOp])->getReg(); @@ -3852,7 +4401,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { assert(!OpInfo.isIndirect && "Don't know how to handle indirect other inputs yet!"); - std::vector Ops; + std::vector Ops; TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode[0], Ops, DAG); if (Ops.empty()) { @@ -3919,34 +4468,41 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // If this asm returns a register value, copy the result from that register // and set it as the value of the call. if (!RetValRegs.Regs.empty()) { - SDOperand Val = RetValRegs.getCopyFromRegs(DAG, Chain, &Flag); - - // If the result of the inline asm is a vector, it may have the wrong - // width/num elts. Make sure to convert it to the right type with + SDValue Val = RetValRegs.getCopyFromRegs(DAG, Chain, &Flag); + + // If any of the results of the inline asm is a vector, it may have the + // wrong width/num elts. This can happen for register classes that can + // contain multiple different value types. The preg or vreg allocated may + // not have the same VT as was expected. Convert it to the right type with // bit_convert. - if (MVT::isVector(Val.getValueType())) { - const VectorType *VTy = cast(CS.getType()); - MVT::ValueType DesiredVT = TLI.getValueType(VTy); - - Val = DAG.getNode(ISD::BIT_CONVERT, DesiredVT, Val); + if (const StructType *ResSTy = dyn_cast(CS.getType())) { + for (unsigned i = 0, e = ResSTy->getNumElements(); i != e; ++i) { + if (Val.Val->getValueType(i).isVector()) + Val = DAG.getNode(ISD::BIT_CONVERT, + TLI.getValueType(ResSTy->getElementType(i)), Val); + } + } else { + if (Val.getValueType().isVector()) + Val = DAG.getNode(ISD::BIT_CONVERT, TLI.getValueType(CS.getType()), + Val); } - + setValue(CS.getInstruction(), Val); } - std::vector > StoresToEmit; + std::vector > StoresToEmit; // Process indirect outputs, first output all of the flagged copies out of // physregs. for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) { RegsForValue &OutRegs = IndirectStoresToEmit[i].first; Value *Ptr = IndirectStoresToEmit[i].second; - SDOperand OutVal = OutRegs.getCopyFromRegs(DAG, Chain, &Flag); + SDValue OutVal = OutRegs.getCopyFromRegs(DAG, Chain, &Flag); StoresToEmit.push_back(std::make_pair(OutVal, Ptr)); } // Emit the non-flagged stores from the physregs. - SmallVector OutChains; + SmallVector OutChains; for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) OutChains.push_back(DAG.getStore(Chain, StoresToEmit[i].first, getValue(StoresToEmit[i].second), @@ -3959,13 +4515,13 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { void SelectionDAGLowering::visitMalloc(MallocInst &I) { - SDOperand Src = getValue(I.getOperand(0)); + SDValue Src = getValue(I.getOperand(0)); - MVT::ValueType IntPtr = TLI.getPointerTy(); + MVT IntPtr = TLI.getPointerTy(); - if (IntPtr < Src.getValueType()) + if (IntPtr.bitsLT(Src.getValueType())) Src = DAG.getNode(ISD::TRUNCATE, IntPtr, Src); - else if (IntPtr > Src.getValueType()) + else if (IntPtr.bitsGT(Src.getValueType())) Src = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, Src); // Scale the source by the type size. @@ -3979,7 +4535,7 @@ void SelectionDAGLowering::visitMalloc(MallocInst &I) { Entry.Ty = TLI.getTargetData()->getIntPtrType(); Args.push_back(Entry); - std::pair Result = + std::pair Result = TLI.LowerCallTo(getRoot(), I.getType(), false, false, false, CallingConv::C, true, DAG.getExternalSymbol("malloc", IntPtr), Args, DAG); setValue(&I, Result.first); // Pointers always fit in registers @@ -3992,8 +4548,8 @@ void SelectionDAGLowering::visitFree(FreeInst &I) { Entry.Node = getValue(I.getOperand(0)); Entry.Ty = TLI.getTargetData()->getIntPtrType(); Args.push_back(Entry); - MVT::ValueType IntPtr = TLI.getPointerTy(); - std::pair Result = + MVT IntPtr = TLI.getPointerTy(); + std::pair Result = TLI.LowerCallTo(getRoot(), Type::VoidTy, false, false, false, CallingConv::C, true, DAG.getExternalSymbol("free", IntPtr), Args, DAG); @@ -4021,7 +4577,7 @@ void SelectionDAGLowering::visitVAStart(CallInst &I) { } void SelectionDAGLowering::visitVAArg(VAArgInst &I) { - SDOperand V = DAG.getVAArg(TLI.getValueType(I.getType()), getRoot(), + SDValue V = DAG.getVAArg(TLI.getValueType(I.getType()), getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0))); setValue(&I, V); @@ -4046,60 +4602,66 @@ void SelectionDAGLowering::visitVACopy(CallInst &I) { /// implementation, which just inserts a FORMAL_ARGUMENTS node. FIXME: When all /// targets are migrated to using FORMAL_ARGUMENTS, this hook should be /// integrated into SDISel. -std::vector -TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { +void TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG, + SmallVectorImpl &ArgValues) { // Add CC# and isVararg as operands to the FORMAL_ARGUMENTS node. - std::vector Ops; + SmallVector Ops; Ops.push_back(DAG.getRoot()); Ops.push_back(DAG.getConstant(F.getCallingConv(), getPointerTy())); Ops.push_back(DAG.getConstant(F.isVarArg(), getPointerTy())); // Add one result value for each formal argument. - std::vector RetVals; + SmallVector RetVals; unsigned j = 1; for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++j) { - MVT::ValueType VT = getValueType(I->getType()); - unsigned Flags = ISD::ParamFlags::NoFlagSet; - unsigned OriginalAlignment = - getTargetData()->getABITypeAlignment(I->getType()); - - // FIXME: Distinguish between a formal with no [sz]ext attribute from one - // that is zero extended! - if (F.paramHasAttr(j, ParamAttr::ZExt)) - Flags &= ~(ISD::ParamFlags::SExt); - if (F.paramHasAttr(j, ParamAttr::SExt)) - Flags |= ISD::ParamFlags::SExt; - if (F.paramHasAttr(j, ParamAttr::InReg)) - Flags |= ISD::ParamFlags::InReg; - if (F.paramHasAttr(j, ParamAttr::StructRet)) - Flags |= ISD::ParamFlags::StructReturn; - if (F.paramHasAttr(j, ParamAttr::ByVal)) { - Flags |= ISD::ParamFlags::ByVal; - const PointerType *Ty = cast(I->getType()); - const Type *ElementTy = Ty->getElementType(); - unsigned FrameAlign = Log2_32(getByValTypeAlignment(ElementTy)); - unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); - // For ByVal, alignment should be passed from FE. BE will guess if - // this info is not there but there are cases it cannot get right. - if (F.getParamAlignment(j)) - FrameAlign = Log2_32(F.getParamAlignment(j)); - Flags |= (FrameAlign << ISD::ParamFlags::ByValAlignOffs); - Flags |= (FrameSize << ISD::ParamFlags::ByValSizeOffs); - } - if (F.paramHasAttr(j, ParamAttr::Nest)) - Flags |= ISD::ParamFlags::Nest; - Flags |= (OriginalAlignment << ISD::ParamFlags::OrigAlignmentOffs); - - MVT::ValueType RegisterVT = getRegisterType(VT); - unsigned NumRegs = getNumRegisters(VT); - for (unsigned i = 0; i != NumRegs; ++i) { - RetVals.push_back(RegisterVT); - // if it isn't first piece, alignment must be 1 - if (i > 0) - Flags = (Flags & (~ISD::ParamFlags::OrigAlignment)) | - (1 << ISD::ParamFlags::OrigAlignmentOffs); - Ops.push_back(DAG.getConstant(Flags, MVT::i32)); + SmallVector ValueVTs; + ComputeValueVTs(*this, I->getType(), ValueVTs); + for (unsigned Value = 0, NumValues = ValueVTs.size(); + Value != NumValues; ++Value) { + MVT VT = ValueVTs[Value]; + const Type *ArgTy = VT.getTypeForMVT(); + ISD::ArgFlagsTy Flags; + unsigned OriginalAlignment = + getTargetData()->getABITypeAlignment(ArgTy); + + if (F.paramHasAttr(j, ParamAttr::ZExt)) + Flags.setZExt(); + if (F.paramHasAttr(j, ParamAttr::SExt)) + Flags.setSExt(); + if (F.paramHasAttr(j, ParamAttr::InReg)) + Flags.setInReg(); + if (F.paramHasAttr(j, ParamAttr::StructRet)) + Flags.setSRet(); + if (F.paramHasAttr(j, ParamAttr::ByVal)) { + Flags.setByVal(); + const PointerType *Ty = cast(I->getType()); + const Type *ElementTy = Ty->getElementType(); + unsigned FrameAlign = getByValTypeAlignment(ElementTy); + unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); + // For ByVal, alignment should be passed from FE. BE will guess if + // this info is not there but there are cases it cannot get right. + if (F.getParamAlignment(j)) + FrameAlign = F.getParamAlignment(j); + Flags.setByValAlign(FrameAlign); + Flags.setByValSize(FrameSize); + } + if (F.paramHasAttr(j, ParamAttr::Nest)) + Flags.setNest(); + Flags.setOrigAlign(OriginalAlignment); + + MVT RegisterVT = getRegisterType(VT); + unsigned NumRegs = getNumRegisters(VT); + for (unsigned i = 0; i != NumRegs; ++i) { + RetVals.push_back(RegisterVT); + ISD::ArgFlagsTy MyFlags = Flags; + if (NumRegs > 1 && i == 0) + MyFlags.setSplit(); + // if it isn't first piece, alignment must be 1 + else if (i > 0) + MyFlags.setOrigAlign(1); + Ops.push_back(DAG.getArgFlags(MyFlags)); + } } } @@ -4113,7 +4675,7 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { // Prelower FORMAL_ARGUMENTS. This isn't required for functionality, but // allows exposing the loads that may be part of the argument access to the // first DAGCombiner pass. - SDOperand TmpRes = LowerOperation(SDOperand(Result, 0), DAG); + SDValue TmpRes = LowerOperation(SDValue(Result, 0), DAG); // The number of results should match up, except that the lowered one may have // an extra flag result. @@ -4121,36 +4683,46 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { (Result->getNumValues()+1 == TmpRes.Val->getNumValues() && TmpRes.getValue(Result->getNumValues()).getValueType() == MVT::Flag)) && "Lowering produced unexpected number of results!"); + + // The FORMAL_ARGUMENTS node itself is likely no longer needed. + if (Result != TmpRes.Val && Result->use_empty()) { + HandleSDNode Dummy(DAG.getRoot()); + DAG.RemoveDeadNode(Result); + } + Result = TmpRes.Val; unsigned NumArgRegs = Result->getNumValues() - 1; - DAG.setRoot(SDOperand(Result, NumArgRegs)); + DAG.setRoot(SDValue(Result, NumArgRegs)); // Set up the return result vector. - Ops.clear(); unsigned i = 0; unsigned Idx = 1; for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++Idx) { - MVT::ValueType VT = getValueType(I->getType()); - MVT::ValueType PartVT = getRegisterType(VT); - - unsigned NumParts = getNumRegisters(VT); - SmallVector Parts(NumParts); - for (unsigned j = 0; j != NumParts; ++j) - Parts[j] = SDOperand(Result, i++); - - ISD::NodeType AssertOp = ISD::DELETED_NODE; - if (F.paramHasAttr(Idx, ParamAttr::SExt)) - AssertOp = ISD::AssertSext; - else if (F.paramHasAttr(Idx, ParamAttr::ZExt)) - AssertOp = ISD::AssertZext; - - Ops.push_back(getCopyFromParts(DAG, &Parts[0], NumParts, PartVT, VT, - AssertOp)); + SmallVector ValueVTs; + ComputeValueVTs(*this, I->getType(), ValueVTs); + for (unsigned Value = 0, NumValues = ValueVTs.size(); + Value != NumValues; ++Value) { + MVT VT = ValueVTs[Value]; + MVT PartVT = getRegisterType(VT); + + unsigned NumParts = getNumRegisters(VT); + SmallVector Parts(NumParts); + for (unsigned j = 0; j != NumParts; ++j) + Parts[j] = SDValue(Result, i++); + + ISD::NodeType AssertOp = ISD::DELETED_NODE; + if (F.paramHasAttr(Idx, ParamAttr::SExt)) + AssertOp = ISD::AssertSext; + else if (F.paramHasAttr(Idx, ParamAttr::ZExt)) + AssertOp = ISD::AssertZext; + + ArgValues.push_back(getCopyFromParts(DAG, &Parts[0], NumParts, PartVT, VT, + AssertOp)); + } } assert(i == NumArgRegs && "Argument register count mismatch!"); - return Ops; } @@ -4158,13 +4730,13 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { /// implementation, which just inserts an ISD::CALL node, which is later custom /// lowered by the target to something concrete. FIXME: When all targets are /// migrated to using ISD::CALL, this hook should be integrated into SDISel. -std::pair -TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, +std::pair +TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, bool RetSExt, bool RetZExt, bool isVarArg, unsigned CallingConv, bool isTailCall, - SDOperand Callee, + SDValue Callee, ArgListTy &Args, SelectionDAG &DAG) { - SmallVector Ops; + SmallVector Ops; Ops.push_back(Chain); // Op#0 - Chain Ops.push_back(DAG.getConstant(CallingConv, getPointerTy())); // Op#1 - CC Ops.push_back(DAG.getConstant(isVarArg, getPointerTy())); // Op#2 - VarArg @@ -4173,76 +4745,91 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, // Handle all of the outgoing arguments. for (unsigned i = 0, e = Args.size(); i != e; ++i) { - MVT::ValueType VT = getValueType(Args[i].Ty); - SDOperand Op = Args[i].Node; - unsigned Flags = ISD::ParamFlags::NoFlagSet; - unsigned OriginalAlignment = - getTargetData()->getABITypeAlignment(Args[i].Ty); - - if (Args[i].isSExt) - Flags |= ISD::ParamFlags::SExt; - if (Args[i].isZExt) - Flags |= ISD::ParamFlags::ZExt; - if (Args[i].isInReg) - Flags |= ISD::ParamFlags::InReg; - if (Args[i].isSRet) - Flags |= ISD::ParamFlags::StructReturn; - if (Args[i].isByVal) { - Flags |= ISD::ParamFlags::ByVal; - const PointerType *Ty = cast(Args[i].Ty); - const Type *ElementTy = Ty->getElementType(); - unsigned FrameAlign = Log2_32(getByValTypeAlignment(ElementTy)); - unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); - // For ByVal, alignment should come from FE. BE will guess if this - // info is not there but there are cases it cannot get right. - if (Args[i].Alignment) - FrameAlign = Log2_32(Args[i].Alignment); - Flags |= (FrameAlign << ISD::ParamFlags::ByValAlignOffs); - Flags |= (FrameSize << ISD::ParamFlags::ByValSizeOffs); - } - if (Args[i].isNest) - Flags |= ISD::ParamFlags::Nest; - Flags |= OriginalAlignment << ISD::ParamFlags::OrigAlignmentOffs; - - MVT::ValueType PartVT = getRegisterType(VT); - unsigned NumParts = getNumRegisters(VT); - SmallVector Parts(NumParts); - ISD::NodeType ExtendKind = ISD::ANY_EXTEND; - - if (Args[i].isSExt) - ExtendKind = ISD::SIGN_EXTEND; - else if (Args[i].isZExt) - ExtendKind = ISD::ZERO_EXTEND; - - getCopyToParts(DAG, Op, &Parts[0], NumParts, PartVT, ExtendKind); - - for (unsigned i = 0; i != NumParts; ++i) { - // if it isn't first piece, alignment must be 1 - unsigned MyFlags = Flags; - if (i != 0) - MyFlags = (MyFlags & (~ISD::ParamFlags::OrigAlignment)) | - (1 << ISD::ParamFlags::OrigAlignmentOffs); - - Ops.push_back(Parts[i]); - Ops.push_back(DAG.getConstant(MyFlags, MVT::i32)); + SmallVector ValueVTs; + ComputeValueVTs(*this, Args[i].Ty, ValueVTs); + for (unsigned Value = 0, NumValues = ValueVTs.size(); + Value != NumValues; ++Value) { + MVT VT = ValueVTs[Value]; + const Type *ArgTy = VT.getTypeForMVT(); + SDValue Op = SDValue(Args[i].Node.Val, Args[i].Node.ResNo + Value); + ISD::ArgFlagsTy Flags; + unsigned OriginalAlignment = + getTargetData()->getABITypeAlignment(ArgTy); + + if (Args[i].isZExt) + Flags.setZExt(); + if (Args[i].isSExt) + Flags.setSExt(); + if (Args[i].isInReg) + Flags.setInReg(); + if (Args[i].isSRet) + Flags.setSRet(); + if (Args[i].isByVal) { + Flags.setByVal(); + const PointerType *Ty = cast(Args[i].Ty); + const Type *ElementTy = Ty->getElementType(); + unsigned FrameAlign = getByValTypeAlignment(ElementTy); + unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); + // For ByVal, alignment should come from FE. BE will guess if this + // info is not there but there are cases it cannot get right. + if (Args[i].Alignment) + FrameAlign = Args[i].Alignment; + Flags.setByValAlign(FrameAlign); + Flags.setByValSize(FrameSize); + } + if (Args[i].isNest) + Flags.setNest(); + Flags.setOrigAlign(OriginalAlignment); + + MVT PartVT = getRegisterType(VT); + unsigned NumParts = getNumRegisters(VT); + SmallVector Parts(NumParts); + ISD::NodeType ExtendKind = ISD::ANY_EXTEND; + + if (Args[i].isSExt) + ExtendKind = ISD::SIGN_EXTEND; + else if (Args[i].isZExt) + ExtendKind = ISD::ZERO_EXTEND; + + getCopyToParts(DAG, Op, &Parts[0], NumParts, PartVT, ExtendKind); + + for (unsigned i = 0; i != NumParts; ++i) { + // if it isn't first piece, alignment must be 1 + ISD::ArgFlagsTy MyFlags = Flags; + if (NumParts > 1 && i == 0) + MyFlags.setSplit(); + else if (i != 0) + MyFlags.setOrigAlign(1); + + Ops.push_back(Parts[i]); + Ops.push_back(DAG.getArgFlags(MyFlags)); + } } } - // Figure out the result value types. - MVT::ValueType VT = getValueType(RetTy); - MVT::ValueType RegisterVT = getRegisterType(VT); - unsigned NumRegs = getNumRegisters(VT); - SmallVector RetTys(NumRegs); - for (unsigned i = 0; i != NumRegs; ++i) - RetTys[i] = RegisterVT; + // Figure out the result value types. We start by making a list of + // the potentially illegal return value types. + SmallVector LoweredRetTys; + SmallVector RetTys; + ComputeValueVTs(*this, RetTy, RetTys); + + // Then we translate that to a list of legal types. + for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { + MVT VT = RetTys[I]; + MVT RegisterVT = getRegisterType(VT); + unsigned NumRegs = getNumRegisters(VT); + for (unsigned i = 0; i != NumRegs; ++i) + LoweredRetTys.push_back(RegisterVT); + } - RetTys.push_back(MVT::Other); // Always has a chain. + LoweredRetTys.push_back(MVT::Other); // Always has a chain. // Create the CALL node. - SDOperand Res = DAG.getNode(ISD::CALL, - DAG.getVTList(&RetTys[0], NumRegs + 1), + SDValue Res = DAG.getNode(ISD::CALL, + DAG.getVTList(&LoweredRetTys[0], + LoweredRetTys.size()), &Ops[0], Ops.size()); - Chain = Res.getValue(NumRegs); + Chain = Res.getValue(LoweredRetTys.size() - 1); // Gather up the call result into a single value. if (RetTy != Type::VoidTy) { @@ -4253,270 +4840,40 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, else if (RetZExt) AssertOp = ISD::AssertZext; - SmallVector Results(NumRegs); - for (unsigned i = 0; i != NumRegs; ++i) - Results[i] = Res.getValue(i); - Res = getCopyFromParts(DAG, &Results[0], NumRegs, RegisterVT, VT, - AssertOp); + SmallVector ReturnValues; + unsigned RegNo = 0; + for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { + MVT VT = RetTys[I]; + MVT RegisterVT = getRegisterType(VT); + unsigned NumRegs = getNumRegisters(VT); + unsigned RegNoEnd = NumRegs + RegNo; + SmallVector Results; + for (; RegNo != RegNoEnd; ++RegNo) + Results.push_back(Res.getValue(RegNo)); + SDValue ReturnValue = + getCopyFromParts(DAG, &Results[0], NumRegs, RegisterVT, VT, + AssertOp); + ReturnValues.push_back(ReturnValue); + } + Res = DAG.getMergeValues(DAG.getVTList(&RetTys[0], RetTys.size()), + &ReturnValues[0], ReturnValues.size()); } return std::make_pair(Res, Chain); } -SDOperand TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { +SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) { assert(0 && "LowerOperation not implemented for this target!"); abort(); - return SDOperand(); -} - -SDOperand TargetLowering::CustomPromoteOperation(SDOperand Op, - SelectionDAG &DAG) { - assert(0 && "CustomPromoteOperation not implemented for this target!"); - abort(); - return SDOperand(); -} - -/// getMemsetValue - Vectorized representation of the memset value -/// operand. -static SDOperand getMemsetValue(SDOperand Value, MVT::ValueType VT, - SelectionDAG &DAG) { - MVT::ValueType CurVT = VT; - if (ConstantSDNode *C = dyn_cast(Value)) { - uint64_t Val = C->getValue() & 255; - unsigned Shift = 8; - while (CurVT != MVT::i8) { - Val = (Val << Shift) | Val; - Shift <<= 1; - CurVT = (MVT::ValueType)((unsigned)CurVT - 1); - } - return DAG.getConstant(Val, VT); - } else { - Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value); - unsigned Shift = 8; - while (CurVT != MVT::i8) { - Value = - DAG.getNode(ISD::OR, VT, - DAG.getNode(ISD::SHL, VT, Value, - DAG.getConstant(Shift, MVT::i8)), Value); - Shift <<= 1; - CurVT = (MVT::ValueType)((unsigned)CurVT - 1); - } - - return Value; - } -} - -/// getMemsetStringVal - Similar to getMemsetValue. Except this is only -/// used when a memcpy is turned into a memset when the source is a constant -/// string ptr. -static SDOperand getMemsetStringVal(MVT::ValueType VT, - SelectionDAG &DAG, TargetLowering &TLI, - std::string &Str, unsigned Offset) { - uint64_t Val = 0; - unsigned MSB = MVT::getSizeInBits(VT) / 8; - if (TLI.isLittleEndian()) - Offset = Offset + MSB - 1; - for (unsigned i = 0; i != MSB; ++i) { - Val = (Val << 8) | (unsigned char)Str[Offset]; - Offset += TLI.isLittleEndian() ? -1 : 1; - } - return DAG.getConstant(Val, VT); -} - -/// getMemBasePlusOffset - Returns base and offset node for the -static SDOperand getMemBasePlusOffset(SDOperand Base, unsigned Offset, - SelectionDAG &DAG, TargetLowering &TLI) { - MVT::ValueType VT = Base.getValueType(); - return DAG.getNode(ISD::ADD, VT, Base, DAG.getConstant(Offset, VT)); -} - -/// MeetsMaxMemopRequirement - Determines if the number of memory ops required -/// to replace the memset / memcpy is below the threshold. It also returns the -/// types of the sequence of memory ops to perform memset / memcpy. -static bool MeetsMaxMemopRequirement(std::vector &MemOps, - unsigned Limit, uint64_t Size, - unsigned Align, TargetLowering &TLI) { - MVT::ValueType VT; - - if (TLI.allowsUnalignedMemoryAccesses()) { - VT = MVT::i64; - } else { - switch (Align & 7) { - case 0: - VT = MVT::i64; - break; - case 4: - VT = MVT::i32; - break; - case 2: - VT = MVT::i16; - break; - default: - VT = MVT::i8; - break; - } - } - - MVT::ValueType LVT = MVT::i64; - while (!TLI.isTypeLegal(LVT)) - LVT = (MVT::ValueType)((unsigned)LVT - 1); - assert(MVT::isInteger(LVT)); - - if (VT > LVT) - VT = LVT; - - unsigned NumMemOps = 0; - while (Size != 0) { - unsigned VTSize = MVT::getSizeInBits(VT) / 8; - while (VTSize > Size) { - VT = (MVT::ValueType)((unsigned)VT - 1); - VTSize >>= 1; - } - assert(MVT::isInteger(VT)); - - if (++NumMemOps > Limit) - return false; - MemOps.push_back(VT); - Size -= VTSize; - } - - return true; + return SDValue(); } -void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) { - SDOperand Op1 = getValue(I.getOperand(1)); - SDOperand Op2 = getValue(I.getOperand(2)); - SDOperand Op3 = getValue(I.getOperand(3)); - SDOperand Op4 = getValue(I.getOperand(4)); - unsigned Align = (unsigned)cast(Op4)->getValue(); - if (Align == 0) Align = 1; - - // If the source and destination are known to not be aliases, we can - // lower memmove as memcpy. - if (Op == ISD::MEMMOVE) { - uint64_t Size = -1ULL; - if (ConstantSDNode *C = dyn_cast(Op3)) - Size = C->getValue(); - if (AA.alias(I.getOperand(1), Size, I.getOperand(2), Size) == - AliasAnalysis::NoAlias) - Op = ISD::MEMCPY; - } - - if (ConstantSDNode *Size = dyn_cast(Op3)) { - std::vector MemOps; - - // Expand memset / memcpy to a series of load / store ops - // if the size operand falls below a certain threshold. - SmallVector OutChains; - switch (Op) { - default: break; // Do nothing for now. - case ISD::MEMSET: { - if (MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemset(), - Size->getValue(), Align, TLI)) { - unsigned NumMemOps = MemOps.size(); - unsigned Offset = 0; - for (unsigned i = 0; i < NumMemOps; i++) { - MVT::ValueType VT = MemOps[i]; - unsigned VTSize = MVT::getSizeInBits(VT) / 8; - SDOperand Value = getMemsetValue(Op2, VT, DAG); - SDOperand Store = DAG.getStore(getRoot(), Value, - getMemBasePlusOffset(Op1, Offset, DAG, TLI), - I.getOperand(1), Offset); - OutChains.push_back(Store); - Offset += VTSize; - } - } - break; - } - case ISD::MEMCPY: { - if (MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemcpy(), - Size->getValue(), Align, TLI)) { - unsigned NumMemOps = MemOps.size(); - unsigned SrcOff = 0, DstOff = 0, SrcDelta = 0; - GlobalAddressSDNode *G = NULL; - std::string Str; - bool CopyFromStr = false; - - if (Op2.getOpcode() == ISD::GlobalAddress) - G = cast(Op2); - else if (Op2.getOpcode() == ISD::ADD && - Op2.getOperand(0).getOpcode() == ISD::GlobalAddress && - Op2.getOperand(1).getOpcode() == ISD::Constant) { - G = cast(Op2.getOperand(0)); - SrcDelta = cast(Op2.getOperand(1))->getValue(); - } - if (G) { - GlobalVariable *GV = dyn_cast(G->getGlobal()); - if (GV && GV->isConstant()) { - Str = GV->getStringValue(false); - if (!Str.empty()) { - CopyFromStr = true; - SrcOff += SrcDelta; - } - } - } - - for (unsigned i = 0; i < NumMemOps; i++) { - MVT::ValueType VT = MemOps[i]; - unsigned VTSize = MVT::getSizeInBits(VT) / 8; - SDOperand Value, Chain, Store; - - if (CopyFromStr) { - Value = getMemsetStringVal(VT, DAG, TLI, Str, SrcOff); - Chain = getRoot(); - Store = - DAG.getStore(Chain, Value, - getMemBasePlusOffset(Op1, DstOff, DAG, TLI), - I.getOperand(1), DstOff); - } else { - Value = DAG.getLoad(VT, getRoot(), - getMemBasePlusOffset(Op2, SrcOff, DAG, TLI), - I.getOperand(2), SrcOff, false, Align); - Chain = Value.getValue(1); - Store = - DAG.getStore(Chain, Value, - getMemBasePlusOffset(Op1, DstOff, DAG, TLI), - I.getOperand(1), DstOff, false, Align); - } - OutChains.push_back(Store); - SrcOff += VTSize; - DstOff += VTSize; - } - } - break; - } - } - - if (!OutChains.empty()) { - DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other, - &OutChains[0], OutChains.size())); - return; - } - } - - SDOperand AlwaysInline = DAG.getConstant(0, MVT::i1); - SDOperand Node; - switch(Op) { - default: - assert(0 && "Unknown Op"); - case ISD::MEMCPY: - Node = DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline); - break; - case ISD::MEMMOVE: - Node = DAG.getMemmove(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline); - break; - case ISD::MEMSET: - Node = DAG.getMemset(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline); - break; - } - DAG.setRoot(Node); -} //===----------------------------------------------------------------------===// // SelectionDAGISel code //===----------------------------------------------------------------------===// -unsigned SelectionDAGISel::MakeReg(MVT::ValueType VT) { +unsigned SelectionDAGISel::MakeReg(MVT VT) { return RegInfo->createVirtualRegister(TLI.getRegClassFor(VT)); } @@ -4526,8 +4883,6 @@ void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); } - - bool SelectionDAGISel::runOnFunction(Function &Fn) { // Get alias analysis for load/store combining. AA = &getAnalysis(); @@ -4542,14 +4897,12 @@ bool SelectionDAGISel::runOnFunction(Function &Fn) { FunctionLoweringInfo FuncInfo(TLI, Fn, MF); - if (ExceptionHandling) - for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) - if (InvokeInst *Invoke = dyn_cast(I->getTerminator())) - // Mark landing pad. - FuncInfo.MBBMap[Invoke->getSuccessor(1)]->setIsLandingPad(); - for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) - SelectBasicBlock(I, MF, FuncInfo); + if (InvokeInst *Invoke = dyn_cast(I->getTerminator())) + // Mark landing pad. + FuncInfo.MBBMap[Invoke->getSuccessor(1)]->setIsLandingPad(); + + SelectAllBasicBlocks(Fn, MF, FuncInfo); // Add function live-ins to entry block live-in set. BasicBlock *EntryBB = &Fn.getEntryBlock(); @@ -4567,49 +4920,45 @@ bool SelectionDAGISel::runOnFunction(Function &Fn) { return true; } -SDOperand SelectionDAGLowering::CopyValueToVirtualRegister(Value *V, - unsigned Reg) { - SDOperand Op = getValue(V); +void SelectionDAGLowering::CopyValueToVirtualRegister(Value *V, unsigned Reg) { + SDValue Op = getValue(V); assert((Op.getOpcode() != ISD::CopyFromReg || cast(Op.getOperand(1))->getReg() != Reg) && "Copy from a reg to the same reg!"); - - MVT::ValueType SrcVT = Op.getValueType(); - MVT::ValueType RegisterVT = TLI.getRegisterType(SrcVT); - unsigned NumRegs = TLI.getNumRegisters(SrcVT); - SmallVector Regs(NumRegs); - SmallVector Chains(NumRegs); - - // Copy the value by legal parts into sequential virtual registers. - getCopyToParts(DAG, Op, &Regs[0], NumRegs, RegisterVT); - for (unsigned i = 0; i != NumRegs; ++i) - Chains[i] = DAG.getCopyToReg(getRoot(), Reg + i, Regs[i]); - return DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumRegs); + assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg"); + + RegsForValue RFV(TLI, Reg, V->getType()); + SDValue Chain = DAG.getEntryNode(); + RFV.getCopyToRegs(Op, DAG, Chain, 0); + PendingExports.push_back(Chain); } void SelectionDAGISel:: -LowerArguments(BasicBlock *LLVMBB, SelectionDAGLowering &SDL, - std::vector &UnorderedChains) { +LowerArguments(BasicBlock *LLVMBB, SelectionDAGLowering &SDL) { // If this is the entry block, emit arguments. Function &F = *LLVMBB->getParent(); FunctionLoweringInfo &FuncInfo = SDL.FuncInfo; - SDOperand OldRoot = SDL.DAG.getRoot(); - std::vector Args = TLI.LowerArguments(F, SDL.DAG); + SDValue OldRoot = SDL.DAG.getRoot(); + SmallVector Args; + TLI.LowerArguments(F, SDL.DAG, Args); unsigned a = 0; for (Function::arg_iterator AI = F.arg_begin(), E = F.arg_end(); - AI != E; ++AI, ++a) + AI != E; ++AI) { + SmallVector ValueVTs; + ComputeValueVTs(TLI, AI->getType(), ValueVTs); + unsigned NumValues = ValueVTs.size(); if (!AI->use_empty()) { - SDL.setValue(AI, Args[a]); - + SDL.setValue(AI, SDL.DAG.getMergeValues(&Args[a], NumValues)); // If this argument is live outside of the entry block, insert a copy from // whereever we got it to the vreg that other BB's will reference it as. DenseMap::iterator VMI=FuncInfo.ValueMap.find(AI); if (VMI != FuncInfo.ValueMap.end()) { - SDOperand Copy = SDL.CopyValueToVirtualRegister(AI, VMI->second); - UnorderedChains.push_back(Copy); + SDL.CopyValueToVirtualRegister(AI, VMI->second); } } + a += NumValues; + } // Finally, if the target has anything special to do, allow it to do so. // FIXME: this should insert code into the DAG! @@ -4629,12 +4978,46 @@ static void copyCatchInfo(BasicBlock *SrcBB, BasicBlock *DestBB, } } +/// IsFixedFrameObjectWithPosOffset - Check if object is a fixed frame object and +/// whether object offset >= 0. +static bool +IsFixedFrameObjectWithPosOffset(MachineFrameInfo * MFI, SDValue Op) { + if (!isa(Op)) return false; + + FrameIndexSDNode * FrameIdxNode = dyn_cast(Op); + int FrameIdx = FrameIdxNode->getIndex(); + return MFI->isFixedObjectIndex(FrameIdx) && + MFI->getObjectOffset(FrameIdx) >= 0; +} + +/// IsPossiblyOverwrittenArgumentOfTailCall - Check if the operand could +/// possibly be overwritten when lowering the outgoing arguments in a tail +/// call. Currently the implementation of this call is very conservative and +/// assumes all arguments sourcing from FORMAL_ARGUMENTS or a CopyFromReg with +/// virtual registers would be overwritten by direct lowering. +static bool IsPossiblyOverwrittenArgumentOfTailCall(SDValue Op, + MachineFrameInfo * MFI) { + RegisterSDNode * OpReg = NULL; + if (Op.getOpcode() == ISD::FORMAL_ARGUMENTS || + (Op.getOpcode()== ISD::CopyFromReg && + (OpReg = dyn_cast(Op.getOperand(1))) && + (OpReg->getReg() >= TargetRegisterInfo::FirstVirtualRegister)) || + (Op.getOpcode() == ISD::LOAD && + IsFixedFrameObjectWithPosOffset(MFI, Op.getOperand(1))) || + (Op.getOpcode() == ISD::MERGE_VALUES && + Op.getOperand(Op.ResNo).getOpcode() == ISD::LOAD && + IsFixedFrameObjectWithPosOffset(MFI, Op.getOperand(Op.ResNo). + getOperand(1)))) + return true; + return false; +} + /// CheckDAGForTailCallsAndFixThem - This Function looks for CALL nodes in the /// DAG and fixes their tailcall attribute operand. static void CheckDAGForTailCallsAndFixThem(SelectionDAG &DAG, TargetLowering& TLI) { SDNode * Ret = NULL; - SDOperand Terminator = DAG.getRoot(); + SDValue Terminator = DAG.getRoot(); // Find RET node. if (Terminator.getOpcode() == ISD::RET) { @@ -4643,29 +5026,62 @@ static void CheckDAGForTailCallsAndFixThem(SelectionDAG &DAG, // Fix tail call attribute of CALL nodes. for (SelectionDAG::allnodes_iterator BE = DAG.allnodes_begin(), - BI = prior(DAG.allnodes_end()); BI != BE; --BI) { + BI = DAG.allnodes_end(); BI != BE; ) { + --BI; if (BI->getOpcode() == ISD::CALL) { - SDOperand OpRet(Ret, 0); - SDOperand OpCall(static_cast(BI), 0); + SDValue OpRet(Ret, 0); + SDValue OpCall(BI, 0); bool isMarkedTailCall = cast(OpCall.getOperand(3))->getValue() != 0; // If CALL node has tail call attribute set to true and the call is not // eligible (no RET or the target rejects) the attribute is fixed to // false. The TargetLowering::IsEligibleForTailCallOptimization function // must correctly identify tail call optimizable calls. - if (isMarkedTailCall && - (Ret==NULL || - !TLI.IsEligibleForTailCallOptimization(OpCall, OpRet, DAG))) { - SmallVector Ops; + if (!isMarkedTailCall) continue; + if (Ret==NULL || + !TLI.IsEligibleForTailCallOptimization(OpCall, OpRet, DAG)) { + // Not eligible. Mark CALL node as non tail call. + SmallVector Ops; unsigned idx=0; - for(SDNode::op_iterator I =OpCall.Val->op_begin(), - E=OpCall.Val->op_end(); I!=E; I++, idx++) { + for(SDNode::op_iterator I =OpCall.Val->op_begin(), + E = OpCall.Val->op_end(); I != E; I++, idx++) { if (idx!=3) Ops.push_back(*I); - else + else Ops.push_back(DAG.getConstant(false, TLI.getPointerTy())); } DAG.UpdateNodeOperands(OpCall, Ops.begin(), Ops.size()); + } else { + // Look for tail call clobbered arguments. Emit a series of + // copyto/copyfrom virtual register nodes to protect them. + SmallVector Ops; + SDValue Chain = OpCall.getOperand(0), InFlag; + unsigned idx=0; + for(SDNode::op_iterator I = OpCall.Val->op_begin(), + E = OpCall.Val->op_end(); I != E; I++, idx++) { + SDValue Arg = *I; + if (idx > 4 && (idx % 2)) { + bool isByVal = cast(OpCall.getOperand(idx+1))-> + getArgFlags().isByVal(); + MachineFunction &MF = DAG.getMachineFunction(); + MachineFrameInfo *MFI = MF.getFrameInfo(); + if (!isByVal && + IsPossiblyOverwrittenArgumentOfTailCall(Arg, MFI)) { + MVT VT = Arg.getValueType(); + unsigned VReg = MF.getRegInfo(). + createVirtualRegister(TLI.getRegClassFor(VT)); + Chain = DAG.getCopyToReg(Chain, VReg, Arg, InFlag); + InFlag = Chain.getValue(1); + Arg = DAG.getCopyFromReg(Chain, VReg, VT, InFlag); + Chain = Arg.getValue(1); + InFlag = Arg.getValue(2); + } + } + Ops.push_back(Arg); + } + // Link in chain of CopyTo/CopyFromReg. + Ops[0] = Chain; + DAG.UpdateNodeOperands(OpCall, Ops.begin(), Ops.size()); } } } @@ -4676,24 +5092,20 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, FunctionLoweringInfo &FuncInfo) { SelectionDAGLowering SDL(DAG, TLI, *AA, FuncInfo, GCI); - std::vector UnorderedChains; - // Lower any arguments needed in this block if this is the entry block. if (LLVMBB == &LLVMBB->getParent()->getEntryBlock()) - LowerArguments(LLVMBB, SDL, UnorderedChains); + LowerArguments(LLVMBB, SDL); BB = FuncInfo.MBBMap[LLVMBB]; SDL.setCurrentBasicBlock(BB); MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - if (ExceptionHandling && MMI && BB->isLandingPad()) { + if (MMI && BB->isLandingPad()) { // Add a label to mark the beginning of the landing pad. Deletion of the // landing pad can thus be detected via the MachineModuleInfo. unsigned LabelID = MMI->addLandingPad(BB); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, DAG.getEntryNode(), - DAG.getConstant(LabelID, MVT::i32), - DAG.getConstant(1, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::EH_LABEL, DAG.getEntryNode(), LabelID)); // Mark exception register as live in. unsigned Reg = TLI.getExceptionAddressRegister(); @@ -4739,8 +5151,7 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, if (!I->use_empty() && !isa(I) && !isa(I)) { DenseMap::iterator VMI =FuncInfo.ValueMap.find(I); if (VMI != FuncInfo.ValueMap.end()) - UnorderedChains.push_back( - SDL.CopyValueToVirtualRegister(I, VMI->second)); + SDL.CopyValueToVirtualRegister(I, VMI->second); } // Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to @@ -4791,8 +5202,7 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, unsigned &RegOut = ConstantsOut[C]; if (RegOut == 0) { RegOut = FuncInfo.CreateRegForValue(C); - UnorderedChains.push_back( - SDL.CopyValueToVirtualRegister(C, RegOut)); + SDL.CopyValueToVirtualRegister(C, RegOut); } Reg = RegOut; } else { @@ -4802,38 +5212,24 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, FuncInfo.StaticAllocaMap.count(cast(PHIOp)) && "Didn't codegen value into a register!??"); Reg = FuncInfo.CreateRegForValue(PHIOp); - UnorderedChains.push_back( - SDL.CopyValueToVirtualRegister(PHIOp, Reg)); + SDL.CopyValueToVirtualRegister(PHIOp, Reg); } } // Remember that this register needs to added to the machine PHI node as // the input for this MBB. - MVT::ValueType VT = TLI.getValueType(PN->getType()); - unsigned NumRegisters = TLI.getNumRegisters(VT); - for (unsigned i = 0, e = NumRegisters; i != e; ++i) - PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); - } - } - ConstantsOut.clear(); - - // Turn all of the unordered chains into one factored node. - if (!UnorderedChains.empty()) { - SDOperand Root = SDL.getRoot(); - if (Root.getOpcode() != ISD::EntryToken) { - unsigned i = 0, e = UnorderedChains.size(); - for (; i != e; ++i) { - assert(UnorderedChains[i].Val->getNumOperands() > 1); - if (UnorderedChains[i].Val->getOperand(0) == Root) - break; // Don't add the root if we already indirectly depend on it. + SmallVector ValueVTs; + ComputeValueVTs(TLI, PN->getType(), ValueVTs); + for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) { + MVT VT = ValueVTs[vti]; + unsigned NumRegisters = TLI.getNumRegisters(VT); + for (unsigned i = 0, e = NumRegisters; i != e; ++i) + PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); + Reg += NumRegisters; } - - if (i == e) - UnorderedChains.push_back(Root); } - DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other, - &UnorderedChains[0], UnorderedChains.size())); } + ConstantsOut.clear(); // Lower the terminator after the copies are emitted. SDL.visit(*LLVMBB->getTerminator()); @@ -4848,7 +5244,7 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, BitTestCases = SDL.BitTestCases; // Make sure the root of the DAG is up-to-date. - DAG.setRoot(SDL.getRoot()); + DAG.setRoot(SDL.getControlRoot()); // Check whether calls in this block are real tail calls. Fix up CALL nodes // with correct tailcall attribute so that the target can rely on the tailcall @@ -4857,48 +5253,209 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, CheckDAGForTailCallsAndFixThem(DAG, TLI); } +void SelectionDAGISel::ComputeLiveOutVRegInfo(SelectionDAG &DAG) { + SmallPtrSet VisitedNodes; + SmallVector Worklist; + + Worklist.push_back(DAG.getRoot().Val); + + APInt Mask; + APInt KnownZero; + APInt KnownOne; + + while (!Worklist.empty()) { + SDNode *N = Worklist.back(); + Worklist.pop_back(); + + // If we've already seen this node, ignore it. + if (!VisitedNodes.insert(N)) + continue; + + // Otherwise, add all chain operands to the worklist. + for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) + if (N->getOperand(i).getValueType() == MVT::Other) + Worklist.push_back(N->getOperand(i).Val); + + // If this is a CopyToReg with a vreg dest, process it. + if (N->getOpcode() != ISD::CopyToReg) + continue; + + unsigned DestReg = cast(N->getOperand(1))->getReg(); + if (!TargetRegisterInfo::isVirtualRegister(DestReg)) + continue; + + // Ignore non-scalar or non-integer values. + SDValue Src = N->getOperand(2); + MVT SrcVT = Src.getValueType(); + if (!SrcVT.isInteger() || SrcVT.isVector()) + continue; + + unsigned NumSignBits = DAG.ComputeNumSignBits(Src); + Mask = APInt::getAllOnesValue(SrcVT.getSizeInBits()); + DAG.ComputeMaskedBits(Src, Mask, KnownZero, KnownOne); + + // Only install this information if it tells us something. + if (NumSignBits != 1 || KnownZero != 0 || KnownOne != 0) { + DestReg -= TargetRegisterInfo::FirstVirtualRegister; + FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); + if (DestReg >= FLI.LiveOutRegInfo.size()) + FLI.LiveOutRegInfo.resize(DestReg+1); + FunctionLoweringInfo::LiveOutInfo &LOI = FLI.LiveOutRegInfo[DestReg]; + LOI.NumSignBits = NumSignBits; + LOI.KnownOne = NumSignBits; + LOI.KnownZero = NumSignBits; + } + } +} + void SelectionDAGISel::CodeGenAndEmitDAG(SelectionDAG &DAG) { - DOUT << "Lowered selection DAG:\n"; + std::string GroupName; + if (TimePassesIsEnabled) + GroupName = "Instruction Selection and Scheduling"; + std::string BlockName; + if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs || + ViewDAGCombine2 || ViewISelDAGs || ViewSchedDAGs || ViewSUnitDAGs) + BlockName = DAG.getMachineFunction().getFunction()->getName() + ':' + + BB->getBasicBlock()->getName(); + + DOUT << "Initial selection DAG:\n"; DEBUG(DAG.dump()); + if (ViewDAGCombine1) DAG.viewGraph("dag-combine1 input for " + BlockName); + // Run the DAG combiner in pre-legalize mode. - DAG.Combine(false, *AA); + if (TimePassesIsEnabled) { + NamedRegionTimer T("DAG Combining 1", GroupName); + DAG.Combine(false, *AA); + } else { + DAG.Combine(false, *AA); + } DOUT << "Optimized lowered selection DAG:\n"; DEBUG(DAG.dump()); // Second step, hack on the DAG until it only uses operations and types that // the target supports. -#if 0 // Enable this some day. - DAG.LegalizeTypes(); - // Someday even later, enable a dag combine pass here. -#endif - DAG.Legalize(); + if (EnableLegalizeTypes) {// Enable this some day. + if (ViewLegalizeTypesDAGs) DAG.viewGraph("legalize-types input for " + + BlockName); + + if (TimePassesIsEnabled) { + NamedRegionTimer T("Type Legalization", GroupName); + DAG.LegalizeTypes(); + } else { + DAG.LegalizeTypes(); + } + + DOUT << "Type-legalized selection DAG:\n"; + DEBUG(DAG.dump()); + + // TODO: enable a dag combine pass here. + } + + if (ViewLegalizeDAGs) DAG.viewGraph("legalize input for " + BlockName); + + if (TimePassesIsEnabled) { + NamedRegionTimer T("DAG Legalization", GroupName); + DAG.Legalize(); + } else { + DAG.Legalize(); + } DOUT << "Legalized selection DAG:\n"; DEBUG(DAG.dump()); + if (ViewDAGCombine2) DAG.viewGraph("dag-combine2 input for " + BlockName); + // Run the DAG combiner in post-legalize mode. - DAG.Combine(true, *AA); + if (TimePassesIsEnabled) { + NamedRegionTimer T("DAG Combining 2", GroupName); + DAG.Combine(true, *AA); + } else { + DAG.Combine(true, *AA); + } DOUT << "Optimized legalized selection DAG:\n"; DEBUG(DAG.dump()); - if (ViewISelDAGs) DAG.viewGraph(); + if (ViewISelDAGs) DAG.viewGraph("isel input for " + BlockName); + + if (!FastISel && EnableValueProp) + ComputeLiveOutVRegInfo(DAG); // Third, instruction select all of the operations to machine code, adding the // code to the MachineBasicBlock. - InstructionSelectBasicBlock(DAG); + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Selection", GroupName); + InstructionSelect(DAG); + } else { + InstructionSelect(DAG); + } + + DOUT << "Selected selection DAG:\n"; + DEBUG(DAG.dump()); + + if (ViewSchedDAGs) DAG.viewGraph("scheduler input for " + BlockName); + + // Schedule machine code. + ScheduleDAG *Scheduler; + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Scheduling", GroupName); + Scheduler = Schedule(DAG); + } else { + Scheduler = Schedule(DAG); + } + + if (ViewSUnitDAGs) Scheduler->viewGraph(); + + // Emit machine code to BB. This can change 'BB' to the last block being + // inserted into. + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Creation", GroupName); + BB = Scheduler->EmitSchedule(); + } else { + BB = Scheduler->EmitSchedule(); + } + + // Free the scheduler state. + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Scheduling Cleanup", GroupName); + delete Scheduler; + } else { + delete Scheduler; + } + + // Perform target specific isel post processing. + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Selection Post Processing", GroupName); + InstructionSelectPostProcessing(); + } else { + InstructionSelectPostProcessing(); + } DOUT << "Selected machine code:\n"; DEBUG(BB->dump()); } -void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, - FunctionLoweringInfo &FuncInfo) { +void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, MachineFunction &MF, + FunctionLoweringInfo &FuncInfo) { + // Define NodeAllocator here so that memory allocation is reused for + // each basic block. + NodeAllocatorType NodeAllocator; + + for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) + SelectBasicBlock(I, MF, FuncInfo, NodeAllocator); +} + +void +SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, + FunctionLoweringInfo &FuncInfo, + NodeAllocatorType &NodeAllocator) { std::vector > PHINodesToUpdate; { - SelectionDAG DAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG DAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &DAG; // First step, lower LLVM code to some DAG. This DAG may use operations and @@ -4932,7 +5489,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i) { // Lower header first, if it wasn't already lowered if (!BitTestCases[i].Emitted) { - SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG HSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &HSDAG; SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -4945,7 +5504,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, } for (unsigned j = 0, ej = BitTestCases[i].Cases.size(); j != ej; ++j) { - SelectionDAG BSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG BSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &BSDAG; SelectionDAGLowering BSDL(BSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -5002,7 +5563,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, for (unsigned i = 0, e = JTCases.size(); i != e; ++i) { // Lower header first, if it wasn't already lowered if (!JTCases[i].first.Emitted) { - SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG HSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &HSDAG; SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -5014,7 +5577,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, CodeGenAndEmitDAG(HSDAG); } - SelectionDAG JSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG JSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &JSDAG; SelectionDAGLowering JSDL(JSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -5062,7 +5627,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, // If we generated any switch lowering information, build and codegen any // additional DAGs necessary. for (unsigned i = 0, e = SwitchCases.size(); i != e; ++i) { - SelectionDAG SDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG SDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + NodeAllocator); CurDAG = &SDAG; SelectionDAGLowering SDL(SDAG, TLI, *AA, FuncInfo, GCI); @@ -5107,12 +5674,10 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, } -//===----------------------------------------------------------------------===// -/// ScheduleAndEmitDAG - Pick a safe ordering and emit instructions for each +/// Schedule - Pick a safe ordering for instructions for each /// target node in the graph. -void SelectionDAGISel::ScheduleAndEmitDAG(SelectionDAG &DAG) { - if (ViewSchedDAGs) DAG.viewGraph(); - +/// +ScheduleDAG *SelectionDAGISel::Schedule(SelectionDAG &DAG) { RegisterScheduler::FunctionPassCtor Ctor = RegisterScheduler::getDefault(); if (!Ctor) { @@ -5120,12 +5685,10 @@ void SelectionDAGISel::ScheduleAndEmitDAG(SelectionDAG &DAG) { RegisterScheduler::setDefault(Ctor); } - ScheduleDAG *SL = Ctor(this, &DAG, BB); - BB = SL->Run(); - - if (ViewSUnitDAGs) SL->viewGraph(); + ScheduleDAG *Scheduler = Ctor(this, &DAG, BB, FastISel); + Scheduler->Run(); - delete SL; + return Scheduler; } @@ -5142,7 +5705,7 @@ HazardRecognizer *SelectionDAGISel::CreateTargetHazardRecognizer() { /// the dag combiner simplified the 255, we still want to match. RHS is the /// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value /// specified in the .td file (e.g. 255). -bool SelectionDAGISel::CheckAndMask(SDOperand LHS, ConstantSDNode *RHS, +bool SelectionDAGISel::CheckAndMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const { const APInt &ActualMask = RHS->getAPIntValue(); const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS); @@ -5171,7 +5734,7 @@ bool SelectionDAGISel::CheckAndMask(SDOperand LHS, ConstantSDNode *RHS, /// the dag combiner simplified the 255, we still want to match. RHS is the /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value /// specified in the .td file (e.g. 255). -bool SelectionDAGISel::CheckOrMask(SDOperand LHS, ConstantSDNode *RHS, +bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const { const APInt &ActualMask = RHS->getAPIntValue(); const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS); @@ -5205,8 +5768,8 @@ bool SelectionDAGISel::CheckOrMask(SDOperand LHS, ConstantSDNode *RHS, /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated /// by tblgen. Others should not call it. void SelectionDAGISel:: -SelectInlineAsmMemoryOperands(std::vector &Ops, SelectionDAG &DAG) { - std::vector InOps; +SelectInlineAsmMemoryOperands(std::vector &Ops, SelectionDAG &DAG) { + std::vector InOps; std::swap(InOps, Ops); Ops.push_back(InOps[0]); // input chain. @@ -5225,14 +5788,14 @@ SelectInlineAsmMemoryOperands(std::vector &Ops, SelectionDAG &DAG) { } else { assert((Flags >> 3) == 1 && "Memory operand with multiple values?"); // Otherwise, this is a memory operand. Ask the target to select it. - std::vector SelOps; + std::vector SelOps; if (SelectInlineAsmMemoryOperand(InOps[i+1], 'm', SelOps, DAG)) { cerr << "Could not match memory address. Inline asm failure!\n"; exit(1); } // Add this to the output node. - MVT::ValueType IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); + MVT IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); Ops.push_back(DAG.getTargetConstant(4/*MEM*/ | (SelOps.size() << 3), IntPtrTy)); Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());