X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAGISel.cpp;h=4b88f7884f5d8287e2c303520b1d92835395c8e6;hb=5bab78527a946632cd9614daa0b9a82ee7b5e1cc;hp=cce0a3254a0d18db64e8ce5b85ba871259fb1fbb;hpb=ea859be53ca13a1547c4675549946b74dc3c6f41;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index cce0a3254a0..4b88f7884f5 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -2,8 +2,8 @@ // // The LLVM Compiler Infrastructure // -// This file was developed by the LLVM research group and is distributed under -// the University of Illinois Open Source License. See LICENSE.TXT for details. +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // @@ -26,14 +26,15 @@ #include "llvm/Intrinsics.h" #include "llvm/IntrinsicInst.h" #include "llvm/ParameterAttributes.h" -#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/Collector.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineJumpTableInfo.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/CodeGen/SSARegMap.h" #include "llvm/Target/MRegisterInfo.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetFrameInfo.h" @@ -54,8 +55,11 @@ ViewISelDAGs("view-isel-dags", cl::Hidden, static cl::opt ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed")); +static cl::opt +ViewSUnitDAGs("view-sunit-dags", cl::Hidden, + cl::desc("Pop up a window to show SUnit dags after they are processed")); #else -static const bool ViewISelDAGs = 0, ViewSchedDAGs = 0; +static const bool ViewISelDAGs = 0, ViewSchedDAGs = 0, ViewSUnitDAGs = 0; #endif //===---------------------------------------------------------------------===// @@ -73,9 +77,10 @@ MachinePassRegistry RegisterScheduler::Registry; namespace { cl::opt > - ISHeuristic("sched", + ISHeuristic("pre-RA-sched", cl::init(&createDefaultScheduler), - cl::desc("Instruction schedulers available:")); + cl::desc("Instruction schedulers available (before register" + " allocation):")); static RegisterScheduler defaultListDAGScheduler("default", " Best scheduler for the target", @@ -90,7 +95,7 @@ namespace { /// This is needed because values can be promoted into larger registers and /// expanded into multiple smaller registers than the value. struct VISIBILITY_HIDDEN RegsForValue { - /// Regs - This list hold the register (for legal and promoted values) + /// Regs - This list holds the register (for legal and promoted values) /// or register set (for expanded values) that the value should be assigned /// to. std::vector Regs; @@ -117,15 +122,16 @@ namespace { /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from /// this value and returns the result as a ValueVT value. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. + /// If the Flag pointer is NULL, no flag is used. SDOperand getCopyFromRegs(SelectionDAG &DAG, - SDOperand &Chain, SDOperand &Flag) const; + SDOperand &Chain, SDOperand *Flag) const; /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the /// specified value into the registers specified by this object. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. + /// If the Flag pointer is NULL, no flag is used. void getCopyToRegs(SDOperand Val, SelectionDAG &DAG, - SDOperand &Chain, SDOperand &Flag, - MVT::ValueType PtrVT) const; + SDOperand &Chain, SDOperand *Flag) const; /// AddInlineAsmOperands - Add this value to the specified inlineasm node /// operand list. This adds the code marker and includes the number of @@ -162,7 +168,7 @@ namespace llvm { TargetLowering &TLI; Function &Fn; MachineFunction &MF; - SSARegMap *RegMap; + MachineRegisterInfo &RegInfo; FunctionLoweringInfo(TargetLowering &TLI, Function &Fn,MachineFunction &MF); @@ -185,7 +191,7 @@ namespace llvm { #endif unsigned MakeReg(MVT::ValueType VT) { - return RegMap->createVirtualRegister(TLI.getRegClassFor(VT)); + return RegInfo.createVirtualRegister(TLI.getRegClassFor(VT)); } /// isExportedInst - Return true if the specified value is an instruction @@ -204,12 +210,12 @@ namespace llvm { }; } -/// isFilterOrSelector - Return true if this instruction is a call to the -/// eh.filter or the eh.selector intrinsic. -static bool isFilterOrSelector(Instruction *I) { +/// isSelector - Return true if this instruction is a call to the +/// eh.selector intrinsic. +static bool isSelector(Instruction *I) { if (IntrinsicInst *II = dyn_cast(I)) - return II->getIntrinsicID() == Intrinsic::eh_selector - || II->getIntrinsicID() == Intrinsic::eh_filter; + return (II->getIntrinsicID() == Intrinsic::eh_selector_i32 || + II->getIntrinsicID() == Intrinsic::eh_selector_i64); return false; } @@ -240,7 +246,7 @@ static bool isOnlyUsedInEntryBlock(Argument *A) { FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, Function &fn, MachineFunction &mf) - : TLI(tli), Fn(fn), MF(mf), RegMap(MF.getSSARegMap()) { + : TLI(tli), Fn(fn), MF(mf), RegInfo(MF.getRegInfo()) { // Create a vreg for each argument register that is not dead and is used // outside of the entry block for the function. @@ -257,7 +263,7 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, if (AllocaInst *AI = dyn_cast(I)) if (ConstantInt *CUI = dyn_cast(AI->getArraySize())) { const Type *Ty = AI->getAllocatedType(); - uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty); + uint64_t TySize = TLI.getTargetData()->getABITypeSize(Ty); unsigned Align = std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), AI->getAlignment()); @@ -290,15 +296,7 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, if (PN->use_empty()) continue; MVT::ValueType VT = TLI.getValueType(PN->getType()); - unsigned NumRegisters; - if (VT != MVT::Vector) - NumRegisters = TLI.getNumRegisters(VT); - else { - MVT::ValueType VT1,VT2; - NumRegisters = - TLI.getVectorTypeBreakdown(cast(PN->getType()), - VT1, VT2); - } + unsigned NumRegisters = TLI.getNumRegisters(VT); unsigned PHIReg = ValueMap[PN]; assert(PHIReg && "PHI node does not have an assigned virtual register!"); const TargetInstrInfo *TII = TLI.getTargetMachine().getInstrInfo(); @@ -314,53 +312,13 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, unsigned FunctionLoweringInfo::CreateRegForValue(const Value *V) { MVT::ValueType VT = TLI.getValueType(V->getType()); - // The number of multiples of registers that we need, to, e.g., split up - // a <2 x int64> -> 4 x i32 registers. - unsigned NumVectorRegs = 1; - - // If this is a vector type, figure out what type it will decompose into - // and how many of the elements it will use. - if (VT == MVT::Vector) { - const VectorType *PTy = cast(V->getType()); - unsigned NumElts = PTy->getNumElements(); - MVT::ValueType EltTy = TLI.getValueType(PTy->getElementType()); - MVT::ValueType VecTy = MVT::getVectorType(EltTy, NumElts); - - // Divide the input until we get to a supported size. This will always - // end with a scalar if the target doesn't support vectors. - while (NumElts > 1 && !TLI.isTypeLegal(VecTy)) { - NumElts >>= 1; - NumVectorRegs <<= 1; - VecTy = MVT::getVectorType(EltTy, NumElts); - } + unsigned NumRegisters = TLI.getNumRegisters(VT); + MVT::ValueType RegisterVT = TLI.getRegisterType(VT); - // Check that VecTy isn't a 1-element vector. - if (NumElts == 1 && VecTy == MVT::Other) - VT = EltTy; - else - VT = VecTy; - } + unsigned R = MakeReg(RegisterVT); + for (unsigned i = 1; i != NumRegisters; ++i) + MakeReg(RegisterVT); - // The common case is that we will only create one register for this - // value. If we have that case, create and return the virtual register. - unsigned NV = TLI.getNumRegisters(VT); - if (NV == 1) { - // If we are promoting this value, pick the next largest supported type. - MVT::ValueType PromotedType = TLI.getTypeToTransformTo(VT); - unsigned Reg = MakeReg(PromotedType); - // If this is a vector of supported or promoted types (e.g. 4 x i16), - // create all of the registers. - for (unsigned i = 1; i != NumVectorRegs; ++i) - MakeReg(PromotedType); - return Reg; - } - - // If this value is represented with multiple target registers, make sure - // to create enough consecutive registers of the right (smaller) type. - VT = TLI.getTypeToExpandTo(VT); - unsigned R = MakeReg(VT); - for (unsigned i = 1; i != NV*NumVectorRegs; ++i) - MakeReg(VT); return R; } @@ -457,6 +415,7 @@ public: TargetLowering &TLI; SelectionDAG &DAG; const TargetData *TD; + AliasAnalysis &AA; /// SwitchCases - Vector of CaseBlock structures used to communicate /// SwitchInst code generation information. @@ -469,11 +428,16 @@ public: /// FuncInfo - Information about the function as a whole. /// FunctionLoweringInfo &FuncInfo; + + /// GCI - Garbage collection metadata for the function. + CollectorMetadata *GCI; SelectionDAGLowering(SelectionDAG &dag, TargetLowering &tli, - FunctionLoweringInfo &funcinfo) - : TLI(tli), DAG(dag), TD(DAG.getTarget().getTargetData()), - FuncInfo(funcinfo) { + AliasAnalysis &aa, + FunctionLoweringInfo &funcinfo, + CollectorMetadata *gci) + : TLI(tli), DAG(dag), TD(DAG.getTarget().getTargetData()), AA(aa), + FuncInfo(funcinfo), GCI(gci) { } /// getRoot - Return the current virtual root of the Selection DAG. @@ -520,10 +484,6 @@ public: const Value *SV, SDOperand Root, bool isVolatile, unsigned Alignment); - SDOperand getIntPtrConstant(uint64_t Val) { - return DAG.getConstant(Val, TLI.getPointerTy()); - } - SDOperand getValue(const Value *V); void setValue(const Value *V, SDOperand NewN) { @@ -541,11 +501,9 @@ public: unsigned Opc); bool isExportableFromCurrentBlock(Value *V, const BasicBlock *FromBB); void ExportFromCurrentBlock(Value *V); - void LowerCallTo(Instruction &I, - const Type *CalledValueTy, unsigned CallingConv, - bool IsTailCall, SDOperand Callee, unsigned OpIdx, + void LowerCallTo(CallSite CS, SDOperand Callee, bool IsTailCall, MachineBasicBlock *LandingPad = NULL); - + // Terminator instructions. void visitRet(ReturnInst &I); void visitBr(BranchInst &I); @@ -582,36 +540,30 @@ public: void visitInvoke(InvokeInst &I); void visitUnwind(UnwindInst &I); - void visitScalarBinary(User &I, unsigned OpCode); - void visitVectorBinary(User &I, unsigned OpCode); - void visitEitherBinary(User &I, unsigned ScalarOp, unsigned VectorOp); + void visitBinary(User &I, unsigned OpCode); void visitShift(User &I, unsigned Opcode); void visitAdd(User &I) { - if (isa(I.getType())) - visitVectorBinary(I, ISD::VADD); - else if (I.getType()->isFloatingPoint()) - visitScalarBinary(I, ISD::FADD); + if (I.getType()->isFPOrFPVector()) + visitBinary(I, ISD::FADD); else - visitScalarBinary(I, ISD::ADD); + visitBinary(I, ISD::ADD); } void visitSub(User &I); void visitMul(User &I) { - if (isa(I.getType())) - visitVectorBinary(I, ISD::VMUL); - else if (I.getType()->isFloatingPoint()) - visitScalarBinary(I, ISD::FMUL); + if (I.getType()->isFPOrFPVector()) + visitBinary(I, ISD::FMUL); else - visitScalarBinary(I, ISD::MUL); + visitBinary(I, ISD::MUL); } - void visitURem(User &I) { visitScalarBinary(I, ISD::UREM); } - void visitSRem(User &I) { visitScalarBinary(I, ISD::SREM); } - void visitFRem(User &I) { visitScalarBinary(I, ISD::FREM); } - void visitUDiv(User &I) { visitEitherBinary(I, ISD::UDIV, ISD::VUDIV); } - void visitSDiv(User &I) { visitEitherBinary(I, ISD::SDIV, ISD::VSDIV); } - void visitFDiv(User &I) { visitEitherBinary(I, ISD::FDIV, ISD::VSDIV); } - void visitAnd (User &I) { visitEitherBinary(I, ISD::AND, ISD::VAND ); } - void visitOr (User &I) { visitEitherBinary(I, ISD::OR, ISD::VOR ); } - void visitXor (User &I) { visitEitherBinary(I, ISD::XOR, ISD::VXOR ); } + void visitURem(User &I) { visitBinary(I, ISD::UREM); } + void visitSRem(User &I) { visitBinary(I, ISD::SREM); } + void visitFRem(User &I) { visitBinary(I, ISD::FREM); } + void visitUDiv(User &I) { visitBinary(I, ISD::UDIV); } + void visitSDiv(User &I) { visitBinary(I, ISD::SDIV); } + void visitFDiv(User &I) { visitBinary(I, ISD::FDIV); } + void visitAnd (User &I) { visitBinary(I, ISD::AND); } + void visitOr (User &I) { visitBinary(I, ISD::OR); } + void visitXor (User &I) { visitBinary(I, ISD::XOR); } void visitShl (User &I) { visitShift(I, ISD::SHL); } void visitLShr(User &I) { visitShift(I, ISD::SRL); } void visitAShr(User &I) { visitShift(I, ISD::SRA); } @@ -645,7 +597,7 @@ public: void visitStore(StoreInst &I); void visitPHI(PHINode &I) { } // PHI nodes are handled specially. void visitCall(CallInst &I); - void visitInlineAsm(CallInst &I); + void visitInlineAsm(CallSite CS); const char *visitIntrinsicCall(CallInst &I, unsigned Intrinsic); void visitTargetIntrinsic(CallInst &I, unsigned Intrinsic); @@ -667,6 +619,208 @@ public: }; } // end namespace llvm + +/// getCopyFromParts - Create a value that contains the +/// specified legal parts combined into the value they represent. +static SDOperand getCopyFromParts(SelectionDAG &DAG, + const SDOperand *Parts, + unsigned NumParts, + MVT::ValueType PartVT, + MVT::ValueType ValueVT, + ISD::NodeType AssertOp = ISD::DELETED_NODE) { + if (!MVT::isVector(ValueVT) || NumParts == 1) { + SDOperand Val = Parts[0]; + + // If the value was expanded, copy from the top part. + if (NumParts > 1) { + assert(NumParts == 2 && + "Cannot expand to more than 2 elts yet!"); + SDOperand Hi = Parts[1]; + if (!DAG.getTargetLoweringInfo().isLittleEndian()) + std::swap(Val, Hi); + return DAG.getNode(ISD::BUILD_PAIR, ValueVT, Val, Hi); + } + + // Otherwise, if the value was promoted or extended, truncate it to the + // appropriate type. + if (PartVT == ValueVT) + return Val; + + if (MVT::isVector(PartVT)) { + assert(MVT::isVector(ValueVT) && "Unknown vector conversion!"); + return DAG.getNode(ISD::BIT_CONVERT, ValueVT, Val); + } + + if (MVT::isVector(ValueVT)) { + assert(NumParts == 1 && + MVT::getVectorElementType(ValueVT) == PartVT && + MVT::getVectorNumElements(ValueVT) == 1 && + "Only trivial scalar-to-vector conversions should get here!"); + return DAG.getNode(ISD::BUILD_VECTOR, ValueVT, Val); + } + + if (MVT::isInteger(PartVT) && + MVT::isInteger(ValueVT)) { + if (ValueVT < PartVT) { + // For a truncate, see if we have any information to + // indicate whether the truncated bits will always be + // zero or sign-extension. + if (AssertOp != ISD::DELETED_NODE) + Val = DAG.getNode(AssertOp, PartVT, Val, + DAG.getValueType(ValueVT)); + return DAG.getNode(ISD::TRUNCATE, ValueVT, Val); + } else { + return DAG.getNode(ISD::ANY_EXTEND, ValueVT, Val); + } + } + + if (MVT::isFloatingPoint(PartVT) && MVT::isFloatingPoint(ValueVT)) + return DAG.getNode(ISD::FP_ROUND, ValueVT, Val, DAG.getIntPtrConstant(0)); + + if (MVT::getSizeInBits(PartVT) == MVT::getSizeInBits(ValueVT)) + return DAG.getNode(ISD::BIT_CONVERT, ValueVT, Val); + + assert(0 && "Unknown mismatch!"); + } + + // Handle a multi-element vector. + MVT::ValueType IntermediateVT, RegisterVT; + unsigned NumIntermediates; + unsigned NumRegs = + DAG.getTargetLoweringInfo() + .getVectorTypeBreakdown(ValueVT, IntermediateVT, NumIntermediates, + RegisterVT); + + assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); + assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); + assert(RegisterVT == Parts[0].getValueType() && + "Part type doesn't match part!"); + + // Assemble the parts into intermediate operands. + SmallVector Ops(NumIntermediates); + if (NumIntermediates == NumParts) { + // If the register was not expanded, truncate or copy the value, + // as appropriate. + for (unsigned i = 0; i != NumParts; ++i) + Ops[i] = getCopyFromParts(DAG, &Parts[i], 1, + PartVT, IntermediateVT); + } else if (NumParts > 0) { + // If the intermediate type was expanded, build the intermediate operands + // from the parts. + assert(NumParts % NumIntermediates == 0 && + "Must expand into a divisible number of parts!"); + unsigned Factor = NumParts / NumIntermediates; + for (unsigned i = 0; i != NumIntermediates; ++i) + Ops[i] = getCopyFromParts(DAG, &Parts[i * Factor], Factor, + PartVT, IntermediateVT); + } + + // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the intermediate + // operands. + return DAG.getNode(MVT::isVector(IntermediateVT) ? + ISD::CONCAT_VECTORS : + ISD::BUILD_VECTOR, + ValueVT, &Ops[0], NumIntermediates); +} + +/// getCopyToParts - Create a series of nodes that contain the +/// specified value split into legal parts. +static void getCopyToParts(SelectionDAG &DAG, + SDOperand Val, + SDOperand *Parts, + unsigned NumParts, + MVT::ValueType PartVT) { + TargetLowering &TLI = DAG.getTargetLoweringInfo(); + MVT::ValueType PtrVT = TLI.getPointerTy(); + MVT::ValueType ValueVT = Val.getValueType(); + + if (!MVT::isVector(ValueVT) || NumParts == 1) { + // If the value was expanded, copy from the parts. + if (NumParts > 1) { + for (unsigned i = 0; i != NumParts; ++i) + Parts[i] = DAG.getNode(ISD::EXTRACT_ELEMENT, PartVT, Val, + DAG.getConstant(i, PtrVT)); + if (!DAG.getTargetLoweringInfo().isLittleEndian()) + std::reverse(Parts, Parts + NumParts); + return; + } + + // If there is a single part and the types differ, this must be + // a promotion. + if (PartVT != ValueVT) { + if (MVT::isVector(PartVT)) { + assert(MVT::isVector(ValueVT) && + "Not a vector-vector cast?"); + Val = DAG.getNode(ISD::BIT_CONVERT, PartVT, Val); + } else if (MVT::isVector(ValueVT)) { + assert(NumParts == 1 && + MVT::getVectorElementType(ValueVT) == PartVT && + MVT::getVectorNumElements(ValueVT) == 1 && + "Only trivial vector-to-scalar conversions should get here!"); + Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, PartVT, Val, + DAG.getConstant(0, PtrVT)); + } else if (MVT::isInteger(PartVT) && MVT::isInteger(ValueVT)) { + if (PartVT < ValueVT) + Val = DAG.getNode(ISD::TRUNCATE, PartVT, Val); + else + Val = DAG.getNode(ISD::ANY_EXTEND, PartVT, Val); + } else if (MVT::isFloatingPoint(PartVT) && + MVT::isFloatingPoint(ValueVT)) { + Val = DAG.getNode(ISD::FP_EXTEND, PartVT, Val); + } else if (MVT::getSizeInBits(PartVT) == + MVT::getSizeInBits(ValueVT)) { + Val = DAG.getNode(ISD::BIT_CONVERT, PartVT, Val); + } else { + assert(0 && "Unknown mismatch!"); + } + } + Parts[0] = Val; + return; + } + + // Handle a multi-element vector. + MVT::ValueType IntermediateVT, RegisterVT; + unsigned NumIntermediates; + unsigned NumRegs = + DAG.getTargetLoweringInfo() + .getVectorTypeBreakdown(ValueVT, IntermediateVT, NumIntermediates, + RegisterVT); + unsigned NumElements = MVT::getVectorNumElements(ValueVT); + + assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); + assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); + + // Split the vector into intermediate operands. + SmallVector Ops(NumIntermediates); + for (unsigned i = 0; i != NumIntermediates; ++i) + if (MVT::isVector(IntermediateVT)) + Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, + IntermediateVT, Val, + DAG.getConstant(i * (NumElements / NumIntermediates), + PtrVT)); + else + Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, + IntermediateVT, Val, + DAG.getConstant(i, PtrVT)); + + // Split the intermediate operands into legal parts. + if (NumParts == NumIntermediates) { + // If the register was not expanded, promote or copy the value, + // as appropriate. + for (unsigned i = 0; i != NumParts; ++i) + getCopyToParts(DAG, Ops[i], &Parts[i], 1, PartVT); + } else if (NumParts > 0) { + // If the intermediate type was expanded, split each the value into + // legal parts. + assert(NumParts % NumIntermediates == 0 && + "Must expand into a divisible number of parts!"); + unsigned Factor = NumParts / NumIntermediates; + for (unsigned i = 0; i != NumIntermediates; ++i) + getCopyToParts(DAG, Ops[i], &Parts[i * Factor], Factor, PartVT); + } +} + + SDOperand SelectionDAGLowering::getValue(const Value *V) { SDOperand &N = NodeMap[V]; if (N.Val) return N; @@ -687,7 +841,7 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { if (!isa(VTy)) return N = DAG.getNode(ISD::UNDEF, VT); - // Create a VBUILD_VECTOR of undef nodes. + // Create a BUILD_VECTOR of undef nodes. const VectorType *PTy = cast(VTy); unsigned NumElements = PTy->getNumElements(); MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); @@ -696,25 +850,24 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { Ops.assign(NumElements, DAG.getNode(ISD::UNDEF, PVT)); // Create a VConstant node with generic Vector type. - Ops.push_back(DAG.getConstant(NumElements, MVT::i32)); - Ops.push_back(DAG.getValueType(PVT)); - return N = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, + MVT::ValueType VT = MVT::getVectorType(PVT, NumElements); + return N = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } else if (ConstantFP *CFP = dyn_cast(C)) { - return N = DAG.getConstantFP(CFP->getValue(), VT); + return N = DAG.getConstantFP(CFP->getValueAPF(), VT); } else if (const VectorType *PTy = dyn_cast(VTy)) { unsigned NumElements = PTy->getNumElements(); MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); // Now that we know the number and type of the elements, push a // Constant or ConstantFP node onto the ops list for each element of - // the packed constant. + // the vector constant. SmallVector Ops; if (ConstantVector *CP = dyn_cast(C)) { for (unsigned i = 0; i != NumElements; ++i) Ops.push_back(getValue(CP->getOperand(i))); } else { - assert(isa(C) && "Unknown packed constant!"); + assert(isa(C) && "Unknown vector constant!"); SDOperand Op; if (MVT::isFloatingPoint(PVT)) Op = DAG.getConstantFP(0, PVT); @@ -723,10 +876,9 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { Ops.assign(NumElements, Op); } - // Create a VBUILD_VECTOR node with generic Vector type. - Ops.push_back(DAG.getConstant(NumElements, MVT::i32)); - Ops.push_back(DAG.getValueType(PVT)); - return NodeMap[V] = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], + // Create a BUILD_VECTOR node. + MVT::ValueType VT = MVT::getVectorType(PVT, NumElements); + return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } else { // Canonicalize all constant ints to be unsigned. @@ -744,79 +896,17 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { unsigned InReg = FuncInfo.ValueMap[V]; assert(InReg && "Value not in map!"); - // If this type is not legal, make it so now. - if (VT != MVT::Vector) { - if (TLI.getTypeAction(VT) == TargetLowering::Expand) { - // Source must be expanded. This input value is actually coming from the - // register pair InReg and InReg+1. - MVT::ValueType DestVT = TLI.getTypeToExpandTo(VT); - unsigned NumVals = TLI.getNumRegisters(VT); - N = DAG.getCopyFromReg(DAG.getEntryNode(), InReg, DestVT); - if (NumVals == 1) - N = DAG.getNode(ISD::BIT_CONVERT, VT, N); - else { - assert(NumVals == 2 && "1 to 4 (and more) expansion not implemented!"); - N = DAG.getNode(ISD::BUILD_PAIR, VT, N, - DAG.getCopyFromReg(DAG.getEntryNode(), InReg+1, DestVT)); - } - } else { - MVT::ValueType DestVT = TLI.getTypeToTransformTo(VT); - N = DAG.getCopyFromReg(DAG.getEntryNode(), InReg, DestVT); - if (TLI.getTypeAction(VT) == TargetLowering::Promote) // Promotion case - N = MVT::isFloatingPoint(VT) - ? DAG.getNode(ISD::FP_ROUND, VT, N) - : DAG.getNode(ISD::TRUNCATE, VT, N); - } - } else { - // Otherwise, if this is a vector, make it available as a generic vector - // here. - MVT::ValueType PTyElementVT, PTyLegalElementVT; - const VectorType *PTy = cast(VTy); - unsigned NE = TLI.getVectorTypeBreakdown(PTy, PTyElementVT, - PTyLegalElementVT); - - // Build a VBUILD_VECTOR or VCONCAT_VECTORS with the input registers. - SmallVector Ops; - if (PTyElementVT == PTyLegalElementVT) { - // If the value types are legal, just VBUILD the CopyFromReg nodes. - for (unsigned i = 0; i != NE; ++i) - Ops.push_back(DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, - PTyElementVT)); - } else if (PTyElementVT < PTyLegalElementVT) { - // If the register was promoted, use TRUNCATE or FP_ROUND as appropriate. - for (unsigned i = 0; i != NE; ++i) { - SDOperand Op = DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, - PTyLegalElementVT); - if (MVT::isFloatingPoint(PTyElementVT)) - Op = DAG.getNode(ISD::FP_ROUND, PTyElementVT, Op); - else - Op = DAG.getNode(ISD::TRUNCATE, PTyElementVT, Op); - Ops.push_back(Op); - } - } else { - // If the register was expanded, use BUILD_PAIR. - assert((NE & 1) == 0 && "Must expand into a multiple of 2 elements!"); - for (unsigned i = 0; i != NE; ++i) { - SDOperand Op0 = DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, - PTyLegalElementVT); - SDOperand Op1 = DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, - PTyLegalElementVT); - Ops.push_back(DAG.getNode(ISD::BUILD_PAIR, PTyElementVT, Op0, Op1)); - } - } - - if (MVT::isVector(PTyElementVT)) { - Ops.push_back(DAG.getConstant(NE * MVT::getVectorNumElements(PTyElementVT), MVT::i32)); - Ops.push_back(DAG.getValueType(MVT::getVectorElementType(PTyElementVT))); - N = DAG.getNode(ISD::VCONCAT_VECTORS, MVT::Vector, &Ops[0], Ops.size()); - } else { - Ops.push_back(DAG.getConstant(NE, MVT::i32)); - Ops.push_back(DAG.getValueType(PTyElementVT)); - N = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], Ops.size()); - } - } - - return N; + MVT::ValueType RegisterVT = TLI.getRegisterType(VT); + unsigned NumRegs = TLI.getNumRegisters(VT); + + std::vector Regs(NumRegs); + for (unsigned i = 0; i != NumRegs; ++i) + Regs[i] = InReg + i; + + RegsForValue RFV(Regs, RegisterVT, VT); + SDOperand Chain = DAG.getEntryNode(); + + return RFV.getCopyFromRegs(DAG, Chain, NULL); } @@ -831,8 +921,8 @@ void SelectionDAGLowering::visitRet(ReturnInst &I) { SDOperand RetOp = getValue(I.getOperand(i)); // If this is an integer return value, we need to promote it ourselves to - // the full width of a register, since LegalizeOp will use ANY_EXTEND rather - // than sign/zero. + // the full width of a register, since getCopyToParts and Legalize will use + // ANY_EXTEND rather than sign/zero. // FIXME: C calling convention requires the return type to be promoted to // at least 32-bit. But this is not necessary for non-C calling conventions. if (MVT::isInteger(RetOp.getValueType()) && @@ -842,17 +932,26 @@ void SelectionDAGLowering::visitRet(ReturnInst &I) { TmpVT = TLI.getTypeToTransformTo(MVT::i32); else TmpVT = MVT::i32; - const FunctionType *FTy = I.getParent()->getParent()->getFunctionType(); - const ParamAttrsList *Attrs = FTy->getParamAttrs(); + const Function *F = I.getParent()->getParent(); ISD::NodeType ExtendKind = ISD::ANY_EXTEND; - if (Attrs && Attrs->paramHasAttr(0, ParamAttr::SExt)) + if (F->paramHasAttr(0, ParamAttr::SExt)) ExtendKind = ISD::SIGN_EXTEND; - if (Attrs && Attrs->paramHasAttr(0, ParamAttr::ZExt)) + if (F->paramHasAttr(0, ParamAttr::ZExt)) ExtendKind = ISD::ZERO_EXTEND; RetOp = DAG.getNode(ExtendKind, TmpVT, RetOp); + NewValues.push_back(RetOp); + NewValues.push_back(DAG.getConstant(false, MVT::i32)); + } else { + MVT::ValueType VT = RetOp.getValueType(); + unsigned NumParts = TLI.getNumRegisters(VT); + MVT::ValueType PartVT = TLI.getRegisterType(VT); + SmallVector Parts(NumParts); + getCopyToParts(DAG, RetOp, &Parts[0], NumParts, PartVT); + for (unsigned i = 0; i < NumParts; ++i) { + NewValues.push_back(Parts[i]); + NewValues.push_back(DAG.getConstant(false, MVT::i32)); + } } - NewValues.push_back(RetOp); - NewValues.push_back(DAG.getConstant(false, MVT::i32)); } DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, &NewValues[0], NewValues.size())); @@ -1065,7 +1164,6 @@ void SelectionDAGLowering::visitBr(BranchInst &I) { // Update machine-CFG edges. CurMBB->addSuccessor(Succ0MBB); - return; } @@ -1220,7 +1318,7 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, // register so it can be used as an index into the jump table in a // subsequent basic block. This value may be smaller or larger than the // target's pointer type, and therefore require extension or truncating. - if (VT > TLI.getPointerTy()) + if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy())) SwitchOp = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), SUB); else SwitchOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), SUB); @@ -1270,7 +1368,7 @@ void SelectionDAGLowering::visitBitTestHeader(SelectionDAGISel::BitTestBlock &B) ISD::SETUGT); SDOperand ShiftOp; - if (VT > TLI.getShiftAmountTy()) + if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getShiftAmountTy())) ShiftOp = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), SUB); else ShiftOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getShiftAmountTy(), SUB); @@ -1348,11 +1446,10 @@ void SelectionDAGLowering::visitInvoke(InvokeInst &I) { MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)]; MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)]; - LowerCallTo(I, I.getCalledValue()->getType(), - I.getCallingConv(), - false, - getValue(I.getOperand(0)), - 3, LandingPad); + if (isa(I.getCalledValue())) + visitInlineAsm(&I); + else + LowerCallTo(&I, getValue(I.getOperand(0)), false, LandingPad); // If the value of the invoke is used outside of its defining block, make it // available as a virtual register. @@ -1813,11 +1910,13 @@ unsigned SelectionDAGLowering::Clusterify(CaseVector& Cases, SI.getSuccessorValue(i), SMBB)); } - sort(Cases.begin(), Cases.end(), CaseCmp()); + std::sort(Cases.begin(), Cases.end(), CaseCmp()); // Merge case into clusters if (Cases.size()>=2) - for (CaseItr I=Cases.begin(), J=++(Cases.begin()), E=Cases.end(); J!=E; ) { + // Must recompute end() each iteration because it may be + // invalidated by erase if we hold on to it + for (CaseItr I=Cases.begin(), J=++(Cases.begin()); J!=Cases.end(); ) { int64_t nextValue = cast(J->Low)->getSExtValue(); int64_t currentValue = cast(I->High)->getSExtValue(); MachineBasicBlock* nextBB = J->BB; @@ -1910,52 +2009,46 @@ void SelectionDAGLowering::visitSub(User &I) { // -0.0 - X --> fneg const Type *Ty = I.getType(); if (isa(Ty)) { - visitVectorBinary(I, ISD::VSUB); - } else if (Ty->isFloatingPoint()) { + if (ConstantVector *CV = dyn_cast(I.getOperand(0))) { + const VectorType *DestTy = cast(I.getType()); + const Type *ElTy = DestTy->getElementType(); + if (ElTy->isFloatingPoint()) { + unsigned VL = DestTy->getNumElements(); + std::vector NZ(VL, ConstantFP::getNegativeZero(ElTy)); + Constant *CNZ = ConstantVector::get(&NZ[0], NZ.size()); + if (CV == CNZ) { + SDOperand Op2 = getValue(I.getOperand(1)); + setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2)); + return; + } + } + } + } + if (Ty->isFloatingPoint()) { if (ConstantFP *CFP = dyn_cast(I.getOperand(0))) - if (CFP->isExactlyValue(-0.0)) { + if (CFP->isExactlyValue(ConstantFP::getNegativeZero(Ty)->getValueAPF())) { SDOperand Op2 = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2)); return; } - visitScalarBinary(I, ISD::FSUB); - } else - visitScalarBinary(I, ISD::SUB); + } + + visitBinary(I, Ty->isFPOrFPVector() ? ISD::FSUB : ISD::SUB); } -void SelectionDAGLowering::visitScalarBinary(User &I, unsigned OpCode) { +void SelectionDAGLowering::visitBinary(User &I, unsigned OpCode) { SDOperand Op1 = getValue(I.getOperand(0)); SDOperand Op2 = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(OpCode, Op1.getValueType(), Op1, Op2)); } -void -SelectionDAGLowering::visitVectorBinary(User &I, unsigned OpCode) { - assert(isa(I.getType())); - const VectorType *Ty = cast(I.getType()); - SDOperand Typ = DAG.getValueType(TLI.getValueType(Ty->getElementType())); - - setValue(&I, DAG.getNode(OpCode, MVT::Vector, - getValue(I.getOperand(0)), - getValue(I.getOperand(1)), - DAG.getConstant(Ty->getNumElements(), MVT::i32), - Typ)); -} - -void SelectionDAGLowering::visitEitherBinary(User &I, unsigned ScalarOp, - unsigned VectorOp) { - if (isa(I.getType())) - visitVectorBinary(I, VectorOp); - else - visitScalarBinary(I, ScalarOp); -} - void SelectionDAGLowering::visitShift(User &I, unsigned Opcode) { SDOperand Op1 = getValue(I.getOperand(0)); SDOperand Op2 = getValue(I.getOperand(1)); - if (TLI.getShiftAmountTy() < Op2.getValueType()) + if (MVT::getSizeInBits(TLI.getShiftAmountTy()) < + MVT::getSizeInBits(Op2.getValueType())) Op2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Op2); else if (TLI.getShiftAmountTy() > Op2.getValueType()) Op2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Op2); @@ -2033,14 +2126,8 @@ void SelectionDAGLowering::visitSelect(User &I) { SDOperand Cond = getValue(I.getOperand(0)); SDOperand TrueVal = getValue(I.getOperand(1)); SDOperand FalseVal = getValue(I.getOperand(2)); - if (!isa(I.getType())) { - setValue(&I, DAG.getNode(ISD::SELECT, TrueVal.getValueType(), Cond, - TrueVal, FalseVal)); - } else { - setValue(&I, DAG.getNode(ISD::VSELECT, MVT::Vector, Cond, TrueVal, FalseVal, - *(TrueVal.Val->op_end()-2), - *(TrueVal.Val->op_end()-1))); - } + setValue(&I, DAG.getNode(ISD::SELECT, TrueVal.getValueType(), Cond, + TrueVal, FalseVal)); } @@ -2071,7 +2158,7 @@ void SelectionDAGLowering::visitFPTrunc(User &I) { // FPTrunc is never a no-op cast, no need to check SDOperand N = getValue(I.getOperand(0)); MVT::ValueType DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getNode(ISD::FP_ROUND, DestVT, N)); + setValue(&I, DAG.getNode(ISD::FP_ROUND, DestVT, N, DAG.getIntPtrConstant(0))); } void SelectionDAGLowering::visitFPExt(User &I){ @@ -2140,23 +2227,6 @@ void SelectionDAGLowering::visitIntToPtr(User &I) { void SelectionDAGLowering::visitBitCast(User &I) { SDOperand N = getValue(I.getOperand(0)); MVT::ValueType DestVT = TLI.getValueType(I.getType()); - if (DestVT == MVT::Vector) { - // This is a cast to a vector from something else. - // Get information about the output vector. - const VectorType *DestTy = cast(I.getType()); - MVT::ValueType EltVT = TLI.getValueType(DestTy->getElementType()); - setValue(&I, DAG.getNode(ISD::VBIT_CONVERT, DestVT, N, - DAG.getConstant(DestTy->getNumElements(),MVT::i32), - DAG.getValueType(EltVT))); - return; - } - MVT::ValueType SrcVT = N.getValueType(); - if (SrcVT == MVT::Vector) { - // This is a cast from a vctor to something else. - // Get information about the input vector. - setValue(&I, DAG.getNode(ISD::VBIT_CONVERT, DestVT, N)); - return; - } // BitCast assures us that source and destination are the same size so this // is either a BIT_CONVERT or a no-op. @@ -2172,18 +2242,16 @@ void SelectionDAGLowering::visitInsertElement(User &I) { SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), getValue(I.getOperand(2))); - SDOperand Num = *(InVec.Val->op_end()-2); - SDOperand Typ = *(InVec.Val->op_end()-1); - setValue(&I, DAG.getNode(ISD::VINSERT_VECTOR_ELT, MVT::Vector, - InVec, InVal, InIdx, Num, Typ)); + setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, + TLI.getValueType(I.getType()), + InVec, InVal, InIdx)); } void SelectionDAGLowering::visitExtractElement(User &I) { SDOperand InVec = getValue(I.getOperand(0)); SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), getValue(I.getOperand(1))); - SDOperand Typ = *(InVec.Val->op_end()-1); - setValue(&I, DAG.getNode(ISD::VEXTRACT_VECTOR_ELT, + setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, TLI.getValueType(I.getType()), InVec, InIdx)); } @@ -2192,10 +2260,9 @@ void SelectionDAGLowering::visitShuffleVector(User &I) { SDOperand V2 = getValue(I.getOperand(1)); SDOperand Mask = getValue(I.getOperand(2)); - SDOperand Num = *(V1.Val->op_end()-2); - SDOperand Typ = *(V2.Val->op_end()-1); - setValue(&I, DAG.getNode(ISD::VVECTOR_SHUFFLE, MVT::Vector, - V1, V2, Mask, Num, Typ)); + setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE, + TLI.getValueType(I.getType()), + V1, V2, Mask)); } @@ -2212,7 +2279,7 @@ void SelectionDAGLowering::visitGetElementPtr(User &I) { // N = N + Offset uint64_t Offset = TD->getStructLayout(StTy)->getElementOffset(Field); N = DAG.getNode(ISD::ADD, N.getValueType(), N, - getIntPtrConstant(Offset)); + DAG.getIntPtrConstant(Offset)); } Ty = StTy->getElementType(Field); } else { @@ -2222,13 +2289,14 @@ void SelectionDAGLowering::visitGetElementPtr(User &I) { if (ConstantInt *CI = dyn_cast(Idx)) { if (CI->getZExtValue() == 0) continue; uint64_t Offs = - TD->getTypeSize(Ty)*cast(CI)->getSExtValue(); - N = DAG.getNode(ISD::ADD, N.getValueType(), N, getIntPtrConstant(Offs)); + TD->getABITypeSize(Ty)*cast(CI)->getSExtValue(); + N = DAG.getNode(ISD::ADD, N.getValueType(), N, + DAG.getIntPtrConstant(Offs)); continue; } // N = N + Idx * ElementSize; - uint64_t ElementSize = TD->getTypeSize(Ty); + uint64_t ElementSize = TD->getABITypeSize(Ty); SDOperand IdxN = getValue(Idx); // If the index is smaller or larger than intptr_t, truncate or extend @@ -2248,7 +2316,7 @@ void SelectionDAGLowering::visitGetElementPtr(User &I) { continue; } - SDOperand Scale = getIntPtrConstant(ElementSize); + SDOperand Scale = DAG.getIntPtrConstant(ElementSize); IdxN = DAG.getNode(ISD::MUL, N.getValueType(), IdxN, Scale); N = DAG.getNode(ISD::ADD, N.getValueType(), N, IdxN); } @@ -2263,7 +2331,7 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) { return; // getValue will auto-populate this. const Type *Ty = I.getAllocatedType(); - uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty); + uint64_t TySize = TLI.getTargetData()->getABITypeSize(Ty); unsigned Align = std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), I.getAlignment()); @@ -2276,25 +2344,25 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) { AllocSize = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, AllocSize); AllocSize = DAG.getNode(ISD::MUL, IntPtr, AllocSize, - getIntPtrConstant(TySize)); + DAG.getIntPtrConstant(TySize)); - // Handle alignment. If the requested alignment is less than or equal to the - // stack alignment, ignore it and round the size of the allocation up to the - // stack alignment size. If the size is greater than the stack alignment, we - // note this in the DYNAMIC_STACKALLOC node. + // Handle alignment. If the requested alignment is less than or equal to + // the stack alignment, ignore it. If the size is greater than or equal to + // the stack alignment, we note this in the DYNAMIC_STACKALLOC node. unsigned StackAlign = TLI.getTargetMachine().getFrameInfo()->getStackAlignment(); - if (Align <= StackAlign) { + if (Align <= StackAlign) Align = 0; - // Add SA-1 to the size. - AllocSize = DAG.getNode(ISD::ADD, AllocSize.getValueType(), AllocSize, - getIntPtrConstant(StackAlign-1)); - // Mask out the low bits for alignment purposes. - AllocSize = DAG.getNode(ISD::AND, AllocSize.getValueType(), AllocSize, - getIntPtrConstant(~(uint64_t)(StackAlign-1))); - } - SDOperand Ops[] = { getRoot(), AllocSize, getIntPtrConstant(Align) }; + // Round the size of the allocation up to the stack alignment size + // by add SA-1 to the size. + AllocSize = DAG.getNode(ISD::ADD, AllocSize.getValueType(), AllocSize, + DAG.getIntPtrConstant(StackAlign-1)); + // Mask out the low bits for alignment purposes. + AllocSize = DAG.getNode(ISD::AND, AllocSize.getValueType(), AllocSize, + DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1))); + + SDOperand Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) }; const MVT::ValueType *VTs = DAG.getNodeValueTypes(AllocSize.getValueType(), MVT::Other); SDOperand DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, VTs, 2, Ops, 3); @@ -2325,15 +2393,9 @@ SDOperand SelectionDAGLowering::getLoadFrom(const Type *Ty, SDOperand Ptr, const Value *SV, SDOperand Root, bool isVolatile, unsigned Alignment) { - SDOperand L; - if (const VectorType *PTy = dyn_cast(Ty)) { - MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); - L = DAG.getVecLoad(PTy->getNumElements(), PVT, Root, Ptr, - DAG.getSrcValue(SV)); - } else { - L = DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, SV, 0, - isVolatile, Alignment); - } + SDOperand L = + DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, SV, 0, + isVolatile, Alignment); if (isVolatile) DAG.setRoot(L.getValue(1)); @@ -2352,31 +2414,13 @@ void SelectionDAGLowering::visitStore(StoreInst &I) { I.isVolatile(), I.getAlignment())); } -/// IntrinsicCannotAccessMemory - Return true if the specified intrinsic cannot -/// access memory and has no other side effects at all. -static bool IntrinsicCannotAccessMemory(unsigned IntrinsicID) { -#define GET_NO_MEMORY_INTRINSICS -#include "llvm/Intrinsics.gen" -#undef GET_NO_MEMORY_INTRINSICS - return false; -} - -// IntrinsicOnlyReadsMemory - Return true if the specified intrinsic doesn't -// have any side-effects or if it only reads memory. -static bool IntrinsicOnlyReadsMemory(unsigned IntrinsicID) { -#define GET_SIDE_EFFECT_INFO -#include "llvm/Intrinsics.gen" -#undef GET_SIDE_EFFECT_INFO - return false; -} - /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC /// node. void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, unsigned Intrinsic) { - bool HasChain = !IntrinsicCannotAccessMemory(Intrinsic); - bool OnlyLoad = HasChain && IntrinsicOnlyReadsMemory(Intrinsic); - + bool HasChain = !I.doesNotAccessMemory(); + bool OnlyLoad = HasChain && I.onlyReadsMemory(); + // Build the operand list. SmallVector Ops; if (HasChain) { // If this intrinsic has side-effects, chainify it. @@ -2394,17 +2438,6 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, // Add all operands of the call to the operand list. for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i) { SDOperand Op = getValue(I.getOperand(i)); - - // If this is a vector type, force it to the right vector type. - if (Op.getValueType() == MVT::Vector) { - const VectorType *OpTy = cast(I.getOperand(i)->getType()); - MVT::ValueType EltVT = TLI.getValueType(OpTy->getElementType()); - - MVT::ValueType VVT = MVT::getVectorType(EltVT, OpTy->getNumElements()); - assert(VVT != MVT::Other && "Intrinsic uses a non-legal type?"); - Op = DAG.getNode(ISD::VBIT_CONVERT, VVT, Op); - } - assert(TLI.isTypeLegal(Op.getValueType()) && "Intrinsic uses a non-legal type?"); Ops.push_back(Op); @@ -2413,7 +2446,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, std::vector VTs; if (I.getType() != Type::VoidTy) { MVT::ValueType VT = TLI.getValueType(I.getType()); - if (VT == MVT::Vector) { + if (MVT::isVector(VT)) { const VectorType *DestTy = cast(I.getType()); MVT::ValueType EltVT = TLI.getValueType(DestTy->getElementType()); @@ -2450,35 +2483,24 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, } if (I.getType() != Type::VoidTy) { if (const VectorType *PTy = dyn_cast(I.getType())) { - MVT::ValueType EVT = TLI.getValueType(PTy->getElementType()); - Result = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Result, - DAG.getConstant(PTy->getNumElements(), MVT::i32), - DAG.getValueType(EVT)); + MVT::ValueType VT = TLI.getValueType(PTy); + Result = DAG.getNode(ISD::BIT_CONVERT, VT, Result); } setValue(&I, Result); } } -/// ExtractGlobalVariable - If C is a global variable, or a bitcast of one -/// (possibly constant folded), return it. Otherwise return NULL. -static GlobalVariable *ExtractGlobalVariable (Constant *C) { - if (GlobalVariable *GV = dyn_cast(C)) - return GV; - else if (ConstantExpr *CE = dyn_cast(C)) { - if (CE->getOpcode() == Instruction::BitCast) - return dyn_cast(CE->getOperand(0)); - else if (CE->getOpcode() == Instruction::GetElementPtr) { - for (unsigned i = 1, e = CE->getNumOperands(); i != e; ++i) - if (!CE->getOperand(i)->isNullValue()) - return NULL; - return dyn_cast(CE->getOperand(0)); - } - } - return NULL; +/// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V. +static GlobalVariable *ExtractTypeInfo (Value *V) { + V = IntrinsicInst::StripPointerCasts(V); + GlobalVariable *GV = dyn_cast(V); + assert (GV || isa(V) && + "TypeInfo must be a global variable or NULL"); + return GV; } /// addCatchInfo - Extract the personality and type infos from an eh.selector -/// or eh.filter call, and add them to the specified machine basic block. +/// call, and add them to the specified machine basic block. static void addCatchInfo(CallInst &I, MachineModuleInfo *MMI, MachineBasicBlock *MBB) { // Inform the MachineModuleInfo of the personality for this landing pad. @@ -2491,17 +2513,44 @@ static void addCatchInfo(CallInst &I, MachineModuleInfo *MMI, // Gather all the type infos for this landing pad and pass them along to // MachineModuleInfo. std::vector TyInfo; - for (unsigned i = 3, N = I.getNumOperands(); i < N; ++i) { - Constant *C = cast(I.getOperand(i)); - GlobalVariable *GV = ExtractGlobalVariable(C); - assert (GV || isa(C) && - "TypeInfo must be a global variable or NULL"); - TyInfo.push_back(GV); + unsigned N = I.getNumOperands(); + + for (unsigned i = N - 1; i > 2; --i) { + if (ConstantInt *CI = dyn_cast(I.getOperand(i))) { + unsigned FilterLength = CI->getZExtValue(); + unsigned FirstCatch = i + FilterLength + !FilterLength; + assert (FirstCatch <= N && "Invalid filter length"); + + if (FirstCatch < N) { + TyInfo.reserve(N - FirstCatch); + for (unsigned j = FirstCatch; j < N; ++j) + TyInfo.push_back(ExtractTypeInfo(I.getOperand(j))); + MMI->addCatchTypeInfo(MBB, TyInfo); + TyInfo.clear(); + } + + if (!FilterLength) { + // Cleanup. + MMI->addCleanup(MBB); + } else { + // Filter. + TyInfo.reserve(FilterLength - 1); + for (unsigned j = i + 1; j < FirstCatch; ++j) + TyInfo.push_back(ExtractTypeInfo(I.getOperand(j))); + MMI->addFilterTypeInfo(MBB, TyInfo); + TyInfo.clear(); + } + + N = i; + } } - if (I.getCalledFunction()->getIntrinsicID() == Intrinsic::eh_filter) - MMI->addFilterTypeInfo(MBB, TyInfo); - else + + if (N > 3) { + TyInfo.reserve(N - 3); + for (unsigned j = 3; j < N; ++j) + TyInfo.push_back(ExtractTypeInfo(I.getOperand(j))); MMI->addCatchTypeInfo(MBB, TyInfo); + } } /// visitIntrinsicCall - Lower the call to the specified intrinsic function. If @@ -2613,13 +2662,12 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::eh_exception: { - MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - - if (ExceptionHandling && MMI) { - // Mark exception register as live in. - unsigned Reg = TLI.getExceptionAddressRegister(); - if (Reg) CurMBB->addLiveIn(Reg); - + if (ExceptionHandling) { + if (!CurMBB->isLandingPad()) { + // FIXME: Mark exception register as live in. Hack for PR1508. + unsigned Reg = TLI.getExceptionAddressRegister(); + if (Reg) CurMBB->addLiveIn(Reg); + } // Insert the EXCEPTIONADDR instruction. SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other); SDOperand Ops[1]; @@ -2633,24 +2681,26 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return 0; } - case Intrinsic::eh_selector: - case Intrinsic::eh_filter:{ + case Intrinsic::eh_selector_i32: + case Intrinsic::eh_selector_i64: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - + MVT::ValueType VT = (Intrinsic == Intrinsic::eh_selector_i32 ? + MVT::i32 : MVT::i64); + if (ExceptionHandling && MMI) { if (CurMBB->isLandingPad()) addCatchInfo(I, MMI, CurMBB); + else { #ifndef NDEBUG - else FuncInfo.CatchInfoLost.insert(&I); #endif - - // Mark exception selector register as live in. - unsigned Reg = TLI.getExceptionSelectorRegister(); - if (Reg) CurMBB->addLiveIn(Reg); + // FIXME: Mark exception selector register as live in. Hack for PR1508. + unsigned Reg = TLI.getExceptionSelectorRegister(); + if (Reg) CurMBB->addLiveIn(Reg); + } // Insert the EHSELECTION instruction. - SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other); + SDVTList VTs = DAG.getVTList(VT, MVT::Other); SDOperand Ops[2]; Ops[0] = getValue(I.getOperand(1)); Ops[1] = getRoot(); @@ -2658,44 +2708,114 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { setValue(&I, Op); DAG.setRoot(Op.getValue(1)); } else { - setValue(&I, DAG.getConstant(0, TLI.getPointerTy())); + setValue(&I, DAG.getConstant(0, VT)); } return 0; } - - case Intrinsic::eh_typeid_for: { + + case Intrinsic::eh_typeid_for_i32: + case Intrinsic::eh_typeid_for_i64: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); + MVT::ValueType VT = (Intrinsic == Intrinsic::eh_typeid_for_i32 ? + MVT::i32 : MVT::i64); if (MMI) { // Find the type id for the given typeinfo. - Constant *C = cast(I.getOperand(1)); - GlobalVariable *GV = ExtractGlobalVariable(C); - assert (GV || isa(C) && - "TypeInfo must be a global variable or NULL"); + GlobalVariable *GV = ExtractTypeInfo(I.getOperand(1)); unsigned TypeID = MMI->getTypeIDFor(GV); - setValue(&I, DAG.getConstant(TypeID, MVT::i32)); + setValue(&I, DAG.getConstant(TypeID, VT)); } else { - setValue(&I, DAG.getConstant(0, MVT::i32)); + // Return something different to eh_selector. + setValue(&I, DAG.getConstant(1, VT)); } return 0; } - case Intrinsic::sqrt_f32: - case Intrinsic::sqrt_f64: + case Intrinsic::eh_return: { + MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); + + if (MMI && ExceptionHandling) { + MMI->setCallsEHReturn(true); + DAG.setRoot(DAG.getNode(ISD::EH_RETURN, + MVT::Other, + getRoot(), + getValue(I.getOperand(1)), + getValue(I.getOperand(2)))); + } else { + setValue(&I, DAG.getConstant(0, TLI.getPointerTy())); + } + + return 0; + } + + case Intrinsic::eh_unwind_init: { + if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo()) { + MMI->setCallsUnwindInit(true); + } + + return 0; + } + + case Intrinsic::eh_dwarf_cfa: { + if (ExceptionHandling) { + MVT::ValueType VT = getValue(I.getOperand(1)).getValueType(); + SDOperand CfaArg; + if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy())) + CfaArg = DAG.getNode(ISD::TRUNCATE, + TLI.getPointerTy(), getValue(I.getOperand(1))); + else + CfaArg = DAG.getNode(ISD::SIGN_EXTEND, + TLI.getPointerTy(), getValue(I.getOperand(1))); + + SDOperand Offset = DAG.getNode(ISD::ADD, + TLI.getPointerTy(), + DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, + TLI.getPointerTy()), + CfaArg); + setValue(&I, DAG.getNode(ISD::ADD, + TLI.getPointerTy(), + DAG.getNode(ISD::FRAMEADDR, + TLI.getPointerTy(), + DAG.getConstant(0, + TLI.getPointerTy())), + Offset)); + } else { + setValue(&I, DAG.getConstant(0, TLI.getPointerTy())); + } + + return 0; + } + + case Intrinsic::sqrt: setValue(&I, DAG.getNode(ISD::FSQRT, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1)))); return 0; - case Intrinsic::powi_f32: - case Intrinsic::powi_f64: + case Intrinsic::powi: setValue(&I, DAG.getNode(ISD::FPOWI, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1)), getValue(I.getOperand(2)))); return 0; + case Intrinsic::sin: + setValue(&I, DAG.getNode(ISD::FSIN, + getValue(I.getOperand(1)).getValueType(), + getValue(I.getOperand(1)))); + return 0; + case Intrinsic::cos: + setValue(&I, DAG.getNode(ISD::FCOS, + getValue(I.getOperand(1)).getValueType(), + getValue(I.getOperand(1)))); + return 0; + case Intrinsic::pow: + setValue(&I, DAG.getNode(ISD::FPOW, + getValue(I.getOperand(1)).getValueType(), + getValue(I.getOperand(1)), + getValue(I.getOperand(2)))); + return 0; case Intrinsic::pcmarker: { SDOperand Tmp = getValue(I.getOperand(1)); DAG.setRoot(DAG.getNode(ISD::PCMARKER, MVT::Other, getRoot(), Tmp)); @@ -2729,10 +2849,6 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { SDOperand Arg = getValue(I.getOperand(1)); MVT::ValueType Ty = Arg.getValueType(); SDOperand result = DAG.getNode(ISD::CTTZ, Ty, Arg); - if (Ty < MVT::i32) - result = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, result); - else if (Ty > MVT::i32) - result = DAG.getNode(ISD::TRUNCATE, MVT::i32, result); setValue(&I, result); return 0; } @@ -2740,10 +2856,6 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { SDOperand Arg = getValue(I.getOperand(1)); MVT::ValueType Ty = Arg.getValueType(); SDOperand result = DAG.getNode(ISD::CTLZ, Ty, Arg); - if (Ty < MVT::i32) - result = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, result); - else if (Ty > MVT::i32) - result = DAG.getNode(ISD::TRUNCATE, MVT::i32, result); setValue(&I, result); return 0; } @@ -2751,10 +2863,6 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { SDOperand Arg = getValue(I.getOperand(1)); MVT::ValueType Ty = Arg.getValueType(); SDOperand result = DAG.getNode(ISD::CTPOP, Ty, Arg); - if (Ty < MVT::i32) - result = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, result); - else if (Ty > MVT::i32) - result = DAG.getNode(ISD::TRUNCATE, MVT::i32, result); setValue(&I, result); return 0; } @@ -2778,63 +2886,112 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::var_annotation: // Discard annotate attributes return 0; + + case Intrinsic::init_trampoline: { + const Function *F = + cast(IntrinsicInst::StripPointerCasts(I.getOperand(2))); + + SDOperand Ops[6]; + Ops[0] = getRoot(); + Ops[1] = getValue(I.getOperand(1)); + Ops[2] = getValue(I.getOperand(2)); + Ops[3] = getValue(I.getOperand(3)); + Ops[4] = DAG.getSrcValue(I.getOperand(1)); + Ops[5] = DAG.getSrcValue(F); + + SDOperand Tmp = DAG.getNode(ISD::TRAMPOLINE, + DAG.getNodeValueTypes(TLI.getPointerTy(), + MVT::Other), 2, + Ops, 6); + + setValue(&I, Tmp); + DAG.setRoot(Tmp.getValue(1)); + return 0; + } + + case Intrinsic::gcroot: + if (GCI) { + Value *Alloca = I.getOperand(1); + Constant *TypeMap = cast(I.getOperand(2)); + + FrameIndexSDNode *FI = cast(getValue(Alloca).Val); + GCI->addStackRoot(FI->getIndex(), TypeMap); + } + return 0; + + case Intrinsic::gcread: + case Intrinsic::gcwrite: + assert(0 && "Collector failed to lower gcread/gcwrite intrinsics!"); + return 0; + + case Intrinsic::flt_rounds: { + setValue(&I, DAG.getNode(ISD::FLT_ROUNDS, MVT::i32)); + return 0; + } + + case Intrinsic::trap: { + DAG.setRoot(DAG.getNode(ISD::TRAP, MVT::Other, getRoot())); + return 0; + } } } -void SelectionDAGLowering::LowerCallTo(Instruction &I, - const Type *CalledValueTy, - unsigned CallingConv, +void SelectionDAGLowering::LowerCallTo(CallSite CS, SDOperand Callee, bool IsTailCall, - SDOperand Callee, unsigned OpIdx, MachineBasicBlock *LandingPad) { - const PointerType *PT = cast(CalledValueTy); + const PointerType *PT = cast(CS.getCalledValue()->getType()); const FunctionType *FTy = cast(PT->getElementType()); - const ParamAttrsList *Attrs = FTy->getParamAttrs(); MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); unsigned BeginLabel = 0, EndLabel = 0; - + TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; - Args.reserve(I.getNumOperands()); - for (unsigned i = OpIdx, e = I.getNumOperands(); i != e; ++i) { - Value *Arg = I.getOperand(i); - SDOperand ArgNode = getValue(Arg); - Entry.Node = ArgNode; Entry.Ty = Arg->getType(); - - unsigned attrInd = i - OpIdx + 1; - Entry.isSExt = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::SExt); - Entry.isZExt = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::ZExt); - Entry.isInReg = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::InReg); - Entry.isSRet = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::StructRet); + Args.reserve(CS.arg_size()); + for (CallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); + i != e; ++i) { + SDOperand ArgNode = getValue(*i); + Entry.Node = ArgNode; Entry.Ty = (*i)->getType(); + + unsigned attrInd = i - CS.arg_begin() + 1; + Entry.isSExt = CS.paramHasAttr(attrInd, ParamAttr::SExt); + Entry.isZExt = CS.paramHasAttr(attrInd, ParamAttr::ZExt); + Entry.isInReg = CS.paramHasAttr(attrInd, ParamAttr::InReg); + Entry.isSRet = CS.paramHasAttr(attrInd, ParamAttr::StructRet); + Entry.isNest = CS.paramHasAttr(attrInd, ParamAttr::Nest); + Entry.isByVal = CS.paramHasAttr(attrInd, ParamAttr::ByVal); Args.push_back(Entry); } - if (ExceptionHandling && MMI) { + bool MarkTryRange = LandingPad || + // C++ requires special handling of 'nounwind' calls. + (CS.doesNotThrow()); + + if (MarkTryRange && ExceptionHandling && MMI) { // Insert a label before the invoke call to mark the try range. This can be // used to detect deletion of the invoke via the MachineModuleInfo. BeginLabel = MMI->NextLabelID(); DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), DAG.getConstant(BeginLabel, MVT::i32))); } - + std::pair Result = - TLI.LowerCallTo(getRoot(), I.getType(), - Attrs && Attrs->paramHasAttr(0, ParamAttr::SExt), - FTy->isVarArg(), CallingConv, IsTailCall, + TLI.LowerCallTo(getRoot(), CS.getType(), + CS.paramHasAttr(0, ParamAttr::SExt), + FTy->isVarArg(), CS.getCallingConv(), IsTailCall, Callee, Args, DAG); - if (I.getType() != Type::VoidTy) - setValue(&I, Result.first); + if (CS.getType() != Type::VoidTy) + setValue(CS.getInstruction(), Result.first); DAG.setRoot(Result.second); - if (ExceptionHandling && MMI) { + if (MarkTryRange && ExceptionHandling && MMI) { // Insert a label at the end of the invoke call to mark the try range. This // can be used to detect deletion of the invoke via the MachineModuleInfo. EndLabel = MMI->NextLabelID(); DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), DAG.getConstant(EndLabel, MVT::i32))); - // Inform MachineModuleInfo of range. + // Inform MachineModuleInfo of range. MMI->addInvoke(LandingPad, BeginLabel, EndLabel); } } @@ -2843,52 +3000,69 @@ void SelectionDAGLowering::LowerCallTo(Instruction &I, void SelectionDAGLowering::visitCall(CallInst &I) { const char *RenameFn = 0; if (Function *F = I.getCalledFunction()) { - if (F->isDeclaration()) + if (F->isDeclaration()) { if (unsigned IID = F->getIntrinsicID()) { RenameFn = visitIntrinsicCall(I, IID); if (!RenameFn) return; - } else { // Not an LLVM intrinsic. - const std::string &Name = F->getName(); - if (Name[0] == 'c' && (Name == "copysign" || Name == "copysignf")) { - if (I.getNumOperands() == 3 && // Basic sanity checks. - I.getOperand(1)->getType()->isFloatingPoint() && - I.getType() == I.getOperand(1)->getType() && - I.getType() == I.getOperand(2)->getType()) { - SDOperand LHS = getValue(I.getOperand(1)); - SDOperand RHS = getValue(I.getOperand(2)); - setValue(&I, DAG.getNode(ISD::FCOPYSIGN, LHS.getValueType(), - LHS, RHS)); - return; - } - } else if (Name[0] == 'f' && (Name == "fabs" || Name == "fabsf")) { - if (I.getNumOperands() == 2 && // Basic sanity checks. - I.getOperand(1)->getType()->isFloatingPoint() && - I.getType() == I.getOperand(1)->getType()) { - SDOperand Tmp = getValue(I.getOperand(1)); - setValue(&I, DAG.getNode(ISD::FABS, Tmp.getValueType(), Tmp)); - return; - } - } else if (Name[0] == 's' && (Name == "sin" || Name == "sinf")) { - if (I.getNumOperands() == 2 && // Basic sanity checks. - I.getOperand(1)->getType()->isFloatingPoint() && - I.getType() == I.getOperand(1)->getType()) { - SDOperand Tmp = getValue(I.getOperand(1)); - setValue(&I, DAG.getNode(ISD::FSIN, Tmp.getValueType(), Tmp)); - return; - } - } else if (Name[0] == 'c' && (Name == "cos" || Name == "cosf")) { - if (I.getNumOperands() == 2 && // Basic sanity checks. - I.getOperand(1)->getType()->isFloatingPoint() && - I.getType() == I.getOperand(1)->getType()) { - SDOperand Tmp = getValue(I.getOperand(1)); - setValue(&I, DAG.getNode(ISD::FCOS, Tmp.getValueType(), Tmp)); - return; - } + } + } + + // Check for well-known libc/libm calls. If the function is internal, it + // can't be a library call. + unsigned NameLen = F->getNameLen(); + if (!F->hasInternalLinkage() && NameLen) { + const char *NameStr = F->getNameStart(); + if (NameStr[0] == 'c' && + ((NameLen == 8 && !strcmp(NameStr, "copysign")) || + (NameLen == 9 && !strcmp(NameStr, "copysignf")))) { + if (I.getNumOperands() == 3 && // Basic sanity checks. + I.getOperand(1)->getType()->isFloatingPoint() && + I.getType() == I.getOperand(1)->getType() && + I.getType() == I.getOperand(2)->getType()) { + SDOperand LHS = getValue(I.getOperand(1)); + SDOperand RHS = getValue(I.getOperand(2)); + setValue(&I, DAG.getNode(ISD::FCOPYSIGN, LHS.getValueType(), + LHS, RHS)); + return; + } + } else if (NameStr[0] == 'f' && + ((NameLen == 4 && !strcmp(NameStr, "fabs")) || + (NameLen == 5 && !strcmp(NameStr, "fabsf")) || + (NameLen == 5 && !strcmp(NameStr, "fabsl")))) { + if (I.getNumOperands() == 2 && // Basic sanity checks. + I.getOperand(1)->getType()->isFloatingPoint() && + I.getType() == I.getOperand(1)->getType()) { + SDOperand Tmp = getValue(I.getOperand(1)); + setValue(&I, DAG.getNode(ISD::FABS, Tmp.getValueType(), Tmp)); + return; + } + } else if (NameStr[0] == 's' && + ((NameLen == 3 && !strcmp(NameStr, "sin")) || + (NameLen == 4 && !strcmp(NameStr, "sinf")) || + (NameLen == 4 && !strcmp(NameStr, "sinl")))) { + if (I.getNumOperands() == 2 && // Basic sanity checks. + I.getOperand(1)->getType()->isFloatingPoint() && + I.getType() == I.getOperand(1)->getType()) { + SDOperand Tmp = getValue(I.getOperand(1)); + setValue(&I, DAG.getNode(ISD::FSIN, Tmp.getValueType(), Tmp)); + return; + } + } else if (NameStr[0] == 'c' && + ((NameLen == 3 && !strcmp(NameStr, "cos")) || + (NameLen == 4 && !strcmp(NameStr, "cosf")) || + (NameLen == 4 && !strcmp(NameStr, "cosl")))) { + if (I.getNumOperands() == 2 && // Basic sanity checks. + I.getOperand(1)->getType()->isFloatingPoint() && + I.getType() == I.getOperand(1)->getType()) { + SDOperand Tmp = getValue(I.getOperand(1)); + setValue(&I, DAG.getNode(ISD::FCOS, Tmp.getValueType(), Tmp)); + return; } } + } } else if (isa(I.getOperand(0))) { - visitInlineAsm(I); + visitInlineAsm(&I); return; } @@ -2898,98 +3072,52 @@ void SelectionDAGLowering::visitCall(CallInst &I) { else Callee = DAG.getExternalSymbol(RenameFn, TLI.getPointerTy()); - LowerCallTo(I, I.getCalledValue()->getType(), - I.getCallingConv(), - I.isTailCall(), - Callee, - 1); + LowerCallTo(&I, Callee, I.isTailCall()); } +/// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from +/// this value and returns the result as a ValueVT value. This uses +/// Chain/Flag as the input and updates them for the output Chain/Flag. +/// If the Flag pointer is NULL, no flag is used. SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG, - SDOperand &Chain, SDOperand &Flag)const{ - SDOperand Val = DAG.getCopyFromReg(Chain, Regs[0], RegVT, Flag); - Chain = Val.getValue(1); - Flag = Val.getValue(2); - - // If the result was expanded, copy from the top part. - if (Regs.size() > 1) { - assert(Regs.size() == 2 && - "Cannot expand to more than 2 elts yet!"); - SDOperand Hi = DAG.getCopyFromReg(Chain, Regs[1], RegVT, Flag); - Chain = Hi.getValue(1); - Flag = Hi.getValue(2); - if (DAG.getTargetLoweringInfo().isLittleEndian()) - return DAG.getNode(ISD::BUILD_PAIR, ValueVT, Val, Hi); - else - return DAG.getNode(ISD::BUILD_PAIR, ValueVT, Hi, Val); - } - - // Otherwise, if the return value was promoted or extended, truncate it to the - // appropriate type. - if (RegVT == ValueVT) - return Val; - - if (MVT::isVector(RegVT)) { - assert(ValueVT == MVT::Vector && "Unknown vector conversion!"); - return DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Val, - DAG.getConstant(MVT::getVectorNumElements(RegVT), - MVT::i32), - DAG.getValueType(MVT::getVectorElementType(RegVT))); - } - - if (MVT::isInteger(RegVT)) { - if (ValueVT < RegVT) - return DAG.getNode(ISD::TRUNCATE, ValueVT, Val); - else - return DAG.getNode(ISD::ANY_EXTEND, ValueVT, Val); + SDOperand &Chain, SDOperand *Flag)const{ + // Copy the legal parts from the registers. + unsigned NumParts = Regs.size(); + SmallVector Parts(NumParts); + for (unsigned i = 0; i != NumParts; ++i) { + SDOperand Part = Flag ? + DAG.getCopyFromReg(Chain, Regs[i], RegVT, *Flag) : + DAG.getCopyFromReg(Chain, Regs[i], RegVT); + Chain = Part.getValue(1); + if (Flag) + *Flag = Part.getValue(2); + Parts[i] = Part; } - assert(MVT::isFloatingPoint(RegVT) && MVT::isFloatingPoint(ValueVT)); - return DAG.getNode(ISD::FP_ROUND, ValueVT, Val); + // Assemble the legal parts into the final value. + return getCopyFromParts(DAG, &Parts[0], NumParts, RegVT, ValueVT); } /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the /// specified value into the registers specified by this object. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. +/// If the Flag pointer is NULL, no flag is used. void RegsForValue::getCopyToRegs(SDOperand Val, SelectionDAG &DAG, - SDOperand &Chain, SDOperand &Flag, - MVT::ValueType PtrVT) const { - if (Regs.size() == 1) { - // If there is a single register and the types differ, this must be - // a promotion. - if (RegVT != ValueVT) { - if (MVT::isVector(RegVT)) { - assert(Val.getValueType() == MVT::Vector &&"Not a vector-vector cast?"); - Val = DAG.getNode(ISD::VBIT_CONVERT, RegVT, Val); - } else if (MVT::isInteger(RegVT) && MVT::isInteger(Val.getValueType())) { - if (RegVT < ValueVT) - Val = DAG.getNode(ISD::TRUNCATE, RegVT, Val); - else - Val = DAG.getNode(ISD::ANY_EXTEND, RegVT, Val); - } else if (MVT::isFloatingPoint(RegVT) && - MVT::isFloatingPoint(Val.getValueType())) { - Val = DAG.getNode(ISD::FP_EXTEND, RegVT, Val); - } else if (MVT::getSizeInBits(RegVT) == - MVT::getSizeInBits(Val.getValueType())) { - Val = DAG.getNode(ISD::BIT_CONVERT, RegVT, Val); - } else { - assert(0 && "Unknown mismatch!"); - } - } - Chain = DAG.getCopyToReg(Chain, Regs[0], Val, Flag); - Flag = Chain.getValue(1); - } else { - std::vector R(Regs); - if (!DAG.getTargetLoweringInfo().isLittleEndian()) - std::reverse(R.begin(), R.end()); - - for (unsigned i = 0, e = R.size(); i != e; ++i) { - SDOperand Part = DAG.getNode(ISD::EXTRACT_ELEMENT, RegVT, Val, - DAG.getConstant(i, PtrVT)); - Chain = DAG.getCopyToReg(Chain, R[i], Part, Flag); - Flag = Chain.getValue(1); - } + SDOperand &Chain, SDOperand *Flag) const { + // Get the list of the values's legal parts. + unsigned NumParts = Regs.size(); + SmallVector Parts(NumParts); + getCopyToParts(DAG, Val, &Parts[0], NumParts, RegVT); + + // Copy the parts into the registers. + for (unsigned i = 0; i != NumParts; ++i) { + SDOperand Part = Flag ? + DAG.getCopyToReg(Chain, Regs[i], Parts[i], *Flag) : + DAG.getCopyToReg(Chain, Regs[i], Parts[i]); + Chain = Part.getValue(0); + if (Flag) + *Flag = Part.getValue(1); } } @@ -3242,9 +3370,9 @@ GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber, ValueVT = RegVT; // Create the appropriate number of virtual registers. - SSARegMap *RegMap = MF.getSSARegMap(); + MachineRegisterInfo &RegInfo = MF.getRegInfo(); for (; NumRegs; --NumRegs) - Regs.push_back(RegMap->createVirtualRegister(PhysReg.second)); + Regs.push_back(RegInfo.createVirtualRegister(PhysReg.second)); OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs); @@ -3311,8 +3439,8 @@ GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber, /// visitInlineAsm - Handle a call to an InlineAsm object. /// -void SelectionDAGLowering::visitInlineAsm(CallInst &I) { - InlineAsm *IA = cast(I.getOperand(0)); +void SelectionDAGLowering::visitInlineAsm(CallSite CS) { + InlineAsm *IA = cast(CS.getCalledValue()); /// ConstraintOperands - Information about all of the constraints. std::vector ConstraintOperands; @@ -3332,7 +3460,7 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) { // registers, because it will not know to avoid the earlyclobbered output reg. bool SawEarlyClobber = false; - unsigned OpNo = 1; // OpNo - The operand of the CallInst. + unsigned ArgNo = 0; // ArgNo - The argument of the CallInst. for (unsigned i = 0, e = ConstraintInfos.size(); i != e; ++i) { ConstraintOperands.push_back(AsmOperandInfo(ConstraintInfos[i])); AsmOperandInfo &OpInfo = ConstraintOperands.back(); @@ -3345,14 +3473,14 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) { if (!OpInfo.isIndirect) { // The return value of the call is this value. As such, there is no // corresponding argument. - assert(I.getType() != Type::VoidTy && "Bad inline asm!"); - OpVT = TLI.getValueType(I.getType()); + assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); + OpVT = TLI.getValueType(CS.getType()); } else { - OpInfo.CallOperandVal = I.getOperand(OpNo++); + OpInfo.CallOperandVal = CS.getArgument(ArgNo++); } break; case InlineAsm::isInput: - OpInfo.CallOperandVal = I.getOperand(OpNo++); + OpInfo.CallOperandVal = CS.getArgument(ArgNo++); break; case InlineAsm::isClobber: // Nothing to do. @@ -3360,31 +3488,37 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) { } // If this is an input or an indirect output, process the call argument. + // BasicBlocks are labels, currently appearing only in asm's. if (OpInfo.CallOperandVal) { - OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); - const Type *OpTy = OpInfo.CallOperandVal->getType(); - // If this is an indirect operand, the operand is a pointer to the - // accessed type. - if (OpInfo.isIndirect) - OpTy = cast(OpTy)->getElementType(); - - // If OpTy is not a first-class value, it may be a struct/union that we - // can tile with integers. - if (!OpTy->isFirstClassType() && OpTy->isSized()) { - unsigned BitSize = TD->getTypeSizeInBits(OpTy); - switch (BitSize) { - default: break; - case 1: - case 8: - case 16: - case 32: - case 64: - OpTy = IntegerType::get(BitSize); - break; + if (isa(OpInfo.CallOperandVal)) + OpInfo.CallOperand = + DAG.getBasicBlock(FuncInfo.MBBMap[cast(OpInfo.CallOperandVal)]); + else { + OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); + const Type *OpTy = OpInfo.CallOperandVal->getType(); + // If this is an indirect operand, the operand is a pointer to the + // accessed type. + if (OpInfo.isIndirect) + OpTy = cast(OpTy)->getElementType(); + + // If OpTy is not a first-class value, it may be a struct/union that we + // can tile with integers. + if (!OpTy->isFirstClassType() && OpTy->isSized()) { + unsigned BitSize = TD->getTypeSizeInBits(OpTy); + switch (BitSize) { + default: break; + case 1: + case 8: + case 16: + case 32: + case 64: + OpTy = IntegerType::get(BitSize); + break; + } } + + OpVT = TLI.getValueType(OpTy, true); } - - OpVT = TLI.getValueType(OpTy, true); } OpInfo.ConstraintVT = OpVT; @@ -3417,7 +3551,7 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) { // Otherwise, create a stack slot and emit a store to it before the // asm. const Type *Ty = OpVal->getType(); - uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty); + uint64_t TySize = TLI.getTargetData()->getABITypeSize(Ty); unsigned Align = TLI.getTargetData()->getPrefTypeAlignment(Ty); MachineFunction &MF = DAG.getMachineFunction(); int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align); @@ -3497,7 +3631,7 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) { // This is the result value of the call. assert(RetValRegs.Regs.empty() && "Cannot have multiple output constraints yet!"); - assert(I.getType() != Type::VoidTy && "Bad inline asm!"); + assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); RetValRegs = OpInfo.AssignedRegs; } else { IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs, @@ -3545,8 +3679,7 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) { } // Use the produced MatchedRegs object to - MatchedRegs.getCopyToRegs(InOperandVal, DAG, Chain, Flag, - TLI.getPointerTy()); + MatchedRegs.getCopyToRegs(InOperandVal, DAG, Chain, &Flag); MatchedRegs.AddInlineAsmOperands(1 /*REGUSE*/, DAG, AsmNodeOperands); break; } else { @@ -3559,20 +3692,20 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) { assert(!OpInfo.isIndirect && "Don't know how to handle indirect other inputs yet!"); - InOperandVal = TLI.isOperandValidForConstraint(InOperandVal, - OpInfo.ConstraintCode[0], - DAG); - if (!InOperandVal.Val) { + std::vector Ops; + TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode[0], + Ops, DAG); + if (Ops.empty()) { cerr << "Invalid operand for inline asm constraint '" << OpInfo.ConstraintCode << "'!\n"; exit(1); } // Add information to the INLINEASM node to know about this input. - unsigned ResOpType = 3 /*IMM*/ | (1 << 3); + unsigned ResOpType = 3 /*IMM*/ | (Ops.size() << 3); AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, TLI.getPointerTy())); - AsmNodeOperands.push_back(InOperandVal); + AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end()); break; } else if (OpInfo.ConstraintType == TargetLowering::C_Memory) { assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!"); @@ -3597,8 +3730,7 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) { assert(!OpInfo.AssignedRegs.Regs.empty() && "Couldn't allocate input reg!"); - OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, Chain, Flag, - TLI.getPointerTy()); + OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, Chain, &Flag); OpInfo.AssignedRegs.AddInlineAsmOperands(1/*REGUSE*/, DAG, AsmNodeOperands); @@ -3627,22 +3759,19 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) { // If this asm returns a register value, copy the result from that register // and set it as the value of the call. if (!RetValRegs.Regs.empty()) { - SDOperand Val = RetValRegs.getCopyFromRegs(DAG, Chain, Flag); + SDOperand Val = RetValRegs.getCopyFromRegs(DAG, Chain, &Flag); // If the result of the inline asm is a vector, it may have the wrong // width/num elts. Make sure to convert it to the right type with - // vbit_convert. - if (Val.getValueType() == MVT::Vector) { - const VectorType *VTy = cast(I.getType()); - unsigned DesiredNumElts = VTy->getNumElements(); - MVT::ValueType DesiredEltVT = TLI.getValueType(VTy->getElementType()); + // bit_convert. + if (MVT::isVector(Val.getValueType())) { + const VectorType *VTy = cast(CS.getType()); + MVT::ValueType DesiredVT = TLI.getValueType(VTy); - Val = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Val, - DAG.getConstant(DesiredNumElts, MVT::i32), - DAG.getValueType(DesiredEltVT)); + Val = DAG.getNode(ISD::BIT_CONVERT, DesiredVT, Val); } - setValue(&I, Val); + setValue(CS.getInstruction(), Val); } std::vector > StoresToEmit; @@ -3652,7 +3781,7 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) { for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) { RegsForValue &OutRegs = IndirectStoresToEmit[i].first; Value *Ptr = IndirectStoresToEmit[i].second; - SDOperand OutVal = OutRegs.getCopyFromRegs(DAG, Chain, Flag); + SDOperand OutVal = OutRegs.getCopyFromRegs(DAG, Chain, &Flag); StoresToEmit.push_back(std::make_pair(OutVal, Ptr)); } @@ -3680,9 +3809,9 @@ void SelectionDAGLowering::visitMalloc(MallocInst &I) { Src = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, Src); // Scale the source by the type size. - uint64_t ElementSize = TD->getTypeSize(I.getType()->getElementType()); + uint64_t ElementSize = TD->getABITypeSize(I.getType()->getElementType()); Src = DAG.getNode(ISD::MUL, Src.getValueType(), - Src, getIntPtrConstant(ElementSize)); + Src, DAG.getIntPtrConstant(ElementSize)); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; @@ -3753,40 +3882,12 @@ void SelectionDAGLowering::visitVACopy(CallInst &I) { DAG.getSrcValue(I.getOperand(2)))); } -/// ExpandScalarFormalArgs - Recursively expand the formal_argument node, either -/// bit_convert it or join a pair of them with a BUILD_PAIR when appropriate. -static SDOperand ExpandScalarFormalArgs(MVT::ValueType VT, SDNode *Arg, - unsigned &i, SelectionDAG &DAG, - TargetLowering &TLI) { - if (TLI.getTypeAction(VT) != TargetLowering::Expand) - return SDOperand(Arg, i++); - - MVT::ValueType EVT = TLI.getTypeToTransformTo(VT); - unsigned NumVals = MVT::getSizeInBits(VT) / MVT::getSizeInBits(EVT); - if (NumVals == 1) { - return DAG.getNode(ISD::BIT_CONVERT, VT, - ExpandScalarFormalArgs(EVT, Arg, i, DAG, TLI)); - } else if (NumVals == 2) { - SDOperand Lo = ExpandScalarFormalArgs(EVT, Arg, i, DAG, TLI); - SDOperand Hi = ExpandScalarFormalArgs(EVT, Arg, i, DAG, TLI); - if (!TLI.isLittleEndian()) - std::swap(Lo, Hi); - return DAG.getNode(ISD::BUILD_PAIR, VT, Lo, Hi); - } else { - // Value scalarized into many values. Unimp for now. - assert(0 && "Cannot expand i64 -> i16 yet!"); - } - return SDOperand(); -} - /// TargetLowering::LowerArguments - This is the default LowerArguments /// implementation, which just inserts a FORMAL_ARGUMENTS node. FIXME: When all /// targets are migrated to using FORMAL_ARGUMENTS, this hook should be /// integrated into SDISel. std::vector TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { - const FunctionType *FTy = F.getFunctionType(); - const ParamAttrsList *Attrs = FTy->getParamAttrs(); // Add CC# and isVararg as operands to the FORMAL_ARGUMENTS node. std::vector Ops; Ops.push_back(DAG.getRoot()); @@ -3805,14 +3906,25 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { // FIXME: Distinguish between a formal with no [sz]ext attribute from one // that is zero extended! - if (Attrs && Attrs->paramHasAttr(j, ParamAttr::ZExt)) + if (F.paramHasAttr(j, ParamAttr::ZExt)) Flags &= ~(ISD::ParamFlags::SExt); - if (Attrs && Attrs->paramHasAttr(j, ParamAttr::SExt)) + if (F.paramHasAttr(j, ParamAttr::SExt)) Flags |= ISD::ParamFlags::SExt; - if (Attrs && Attrs->paramHasAttr(j, ParamAttr::InReg)) + if (F.paramHasAttr(j, ParamAttr::InReg)) Flags |= ISD::ParamFlags::InReg; - if (Attrs && Attrs->paramHasAttr(j, ParamAttr::StructRet)) + if (F.paramHasAttr(j, ParamAttr::StructRet)) Flags |= ISD::ParamFlags::StructReturn; + if (F.paramHasAttr(j, ParamAttr::ByVal)) { + Flags |= ISD::ParamFlags::ByVal; + const PointerType *Ty = cast(I->getType()); + const Type *ElementTy = Ty->getElementType(); + unsigned FrameAlign = Log2_32(getByValTypeAlignment(ElementTy)); + unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); + Flags |= (FrameAlign << ISD::ParamFlags::ByValAlignOffs); + Flags |= (FrameSize << ISD::ParamFlags::ByValSizeOffs); + } + if (F.paramHasAttr(j, ParamAttr::Nest)) + Flags |= ISD::ParamFlags::Nest; Flags |= (OriginalAlignment << ISD::ParamFlags::OrigAlignmentOffs); switch (getTypeAction(VT)) { @@ -3825,39 +3937,22 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { RetVals.push_back(getTypeToTransformTo(VT)); Ops.push_back(DAG.getConstant(Flags, MVT::i32)); break; - case Expand: - if (VT != MVT::Vector) { - // If this is a large integer, it needs to be broken up into small - // integers. Figure out what the destination type is and how many small - // integers it turns into. - MVT::ValueType NVT = getTypeToExpandTo(VT); - unsigned NumVals = getNumRegisters(VT); - for (unsigned i = 0; i != NumVals; ++i) { - RetVals.push_back(NVT); - // if it isn't first piece, alignment must be 1 - if (i > 0) - Flags = (Flags & (~ISD::ParamFlags::OrigAlignment)) | - (1 << ISD::ParamFlags::OrigAlignmentOffs); - Ops.push_back(DAG.getConstant(Flags, MVT::i32)); - } - } else { - // Otherwise, this is a vector type. We only support legal vectors - // right now. - unsigned NumElems = cast(I->getType())->getNumElements(); - const Type *EltTy = cast(I->getType())->getElementType(); - - // Figure out if there is a Packed type corresponding to this Vector - // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems); - if (TVT != MVT::Other && isTypeLegal(TVT)) { - RetVals.push_back(TVT); - Ops.push_back(DAG.getConstant(Flags, MVT::i32)); - } else { - assert(0 && "Don't support illegal by-val vector arguments yet!"); - } + case Expand: { + // If this is an illegal type, it needs to be broken up to fit into + // registers. + MVT::ValueType RegisterVT = getRegisterType(VT); + unsigned NumRegs = getNumRegisters(VT); + for (unsigned i = 0; i != NumRegs; ++i) { + RetVals.push_back(RegisterVT); + // if it isn't first piece, alignment must be 1 + if (i > 0) + Flags = (Flags & (~ISD::ParamFlags::OrigAlignment)) | + (1 << ISD::ParamFlags::OrigAlignmentOffs); + Ops.push_back(DAG.getConstant(Flags, MVT::i32)); } break; } + } } RetVals.push_back(MVT::Other); @@ -3866,8 +3961,8 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { SDNode *Result = DAG.getNode(ISD::FORMAL_ARGUMENTS, DAG.getNodeValueTypes(RetVals), RetVals.size(), &Ops[0], Ops.size()).Val; - - DAG.setRoot(SDOperand(Result, Result->getNumValues()-1)); + unsigned NumArgRegs = Result->getNumValues() - 1; + DAG.setRoot(SDOperand(Result, NumArgRegs)); // Set up the return result vector. Ops.clear(); @@ -3885,94 +3980,36 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { case Promote: { SDOperand Op(Result, i++); if (MVT::isInteger(VT)) { - if (Attrs && Attrs->paramHasAttr(Idx, ParamAttr::SExt)) + if (F.paramHasAttr(Idx, ParamAttr::SExt)) Op = DAG.getNode(ISD::AssertSext, Op.getValueType(), Op, DAG.getValueType(VT)); - else if (Attrs && Attrs->paramHasAttr(Idx, ParamAttr::ZExt)) + else if (F.paramHasAttr(Idx, ParamAttr::ZExt)) Op = DAG.getNode(ISD::AssertZext, Op.getValueType(), Op, DAG.getValueType(VT)); Op = DAG.getNode(ISD::TRUNCATE, VT, Op); } else { assert(MVT::isFloatingPoint(VT) && "Not int or FP?"); - Op = DAG.getNode(ISD::FP_ROUND, VT, Op); + Op = DAG.getNode(ISD::FP_ROUND, VT, Op, DAG.getIntPtrConstant(1)); } Ops.push_back(Op); break; } - case Expand: - if (VT != MVT::Vector) { - // If this is a large integer or a floating point node that needs to be - // expanded, it needs to be reassembled from small integers. Figure out - // what the source elt type is and how many small integers it is. - Ops.push_back(ExpandScalarFormalArgs(VT, Result, i, DAG, *this)); - } else { - // Otherwise, this is a vector type. We only support legal vectors - // right now. - const VectorType *PTy = cast(I->getType()); - unsigned NumElems = PTy->getNumElements(); - const Type *EltTy = PTy->getElementType(); - - // Figure out if there is a Packed type corresponding to this Vector - // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems); - if (TVT != MVT::Other && isTypeLegal(TVT)) { - SDOperand N = SDOperand(Result, i++); - // Handle copies from generic vectors to registers. - N = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, N, - DAG.getConstant(NumElems, MVT::i32), - DAG.getValueType(getValueType(EltTy))); - Ops.push_back(N); - } else { - assert(0 && "Don't support illegal by-val vector arguments yet!"); - abort(); - } - } + case Expand: { + MVT::ValueType PartVT = getRegisterType(VT); + unsigned NumParts = getNumRegisters(VT); + SmallVector Parts(NumParts); + for (unsigned j = 0; j != NumParts; ++j) + Parts[j] = SDOperand(Result, i++); + Ops.push_back(getCopyFromParts(DAG, &Parts[0], NumParts, PartVT, VT)); break; } + } } + assert(i == NumArgRegs && "Argument register count mismatch!"); return Ops; } -/// ExpandScalarCallArgs - Recursively expand call argument node by -/// bit_converting it or extract a pair of elements from the larger node. -static void ExpandScalarCallArgs(MVT::ValueType VT, SDOperand Arg, - unsigned Flags, - SmallVector &Ops, - SelectionDAG &DAG, - TargetLowering &TLI, - bool isFirst = true) { - - if (TLI.getTypeAction(VT) != TargetLowering::Expand) { - // if it isn't first piece, alignment must be 1 - if (!isFirst) - Flags = (Flags & (~ISD::ParamFlags::OrigAlignment)) | - (1 << ISD::ParamFlags::OrigAlignmentOffs); - Ops.push_back(Arg); - Ops.push_back(DAG.getConstant(Flags, MVT::i32)); - return; - } - - MVT::ValueType EVT = TLI.getTypeToTransformTo(VT); - unsigned NumVals = MVT::getSizeInBits(VT) / MVT::getSizeInBits(EVT); - if (NumVals == 1) { - Arg = DAG.getNode(ISD::BIT_CONVERT, EVT, Arg); - ExpandScalarCallArgs(EVT, Arg, Flags, Ops, DAG, TLI, isFirst); - } else if (NumVals == 2) { - SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, EVT, Arg, - DAG.getConstant(0, TLI.getPointerTy())); - SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, EVT, Arg, - DAG.getConstant(1, TLI.getPointerTy())); - if (!TLI.isLittleEndian()) - std::swap(Lo, Hi); - ExpandScalarCallArgs(EVT, Lo, Flags, Ops, DAG, TLI, isFirst); - ExpandScalarCallArgs(EVT, Hi, Flags, Ops, DAG, TLI, false); - } else { - // Value scalarized into many values. Unimp for now. - assert(0 && "Cannot expand i64 -> i16 yet!"); - } -} - /// TargetLowering::LowerCallTo - This is the default LowerCallTo /// implementation, which just inserts an ISD::CALL node, which is later custom /// lowered by the target to something concrete. FIXME: When all targets are @@ -4006,6 +4043,17 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, Flags |= ISD::ParamFlags::InReg; if (Args[i].isSRet) Flags |= ISD::ParamFlags::StructReturn; + if (Args[i].isByVal) { + Flags |= ISD::ParamFlags::ByVal; + const PointerType *Ty = cast(Args[i].Ty); + const Type *ElementTy = Ty->getElementType(); + unsigned FrameAlign = Log2_32(getByValTypeAlignment(ElementTy)); + unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); + Flags |= (FrameAlign << ISD::ParamFlags::ByValAlignOffs); + Flags |= (FrameSize << ISD::ParamFlags::ByValSizeOffs); + } + if (Args[i].isNest) + Flags |= ISD::ParamFlags::Nest; Flags |= OriginalAlignment << ISD::ParamFlags::OrigAlignmentOffs; switch (getTypeAction(VT)) { @@ -4026,152 +4074,59 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, Op = DAG.getNode(ExtOp, getTypeToTransformTo(VT), Op); } else { assert(MVT::isFloatingPoint(VT) && "Not int or FP?"); - // A true promotion would change the size of the argument. - // Instead, pretend this is an int. If FP objects are not - // passed the same as ints, the original type should be Legal - // and we should not get here. - Op = DAG.getNode(ISD::BIT_CONVERT, - VT==MVT::f32 ? MVT::i32 : - (VT==MVT::f64 ? MVT::i64 : - MVT::Other), - Op); + Op = DAG.getNode(ISD::FP_EXTEND, getTypeToTransformTo(VT), Op); } Ops.push_back(Op); Ops.push_back(DAG.getConstant(Flags, MVT::i32)); break; - case Expand: - if (VT != MVT::Vector) { - // If this is a large integer, it needs to be broken down into small - // integers. Figure out what the source elt type is and how many small - // integers it is. - ExpandScalarCallArgs(VT, Op, Flags, Ops, DAG, *this); - } else { - // Otherwise, this is a vector type. We only support legal vectors - // right now. - const VectorType *PTy = cast(Args[i].Ty); - unsigned NumElems = PTy->getNumElements(); - const Type *EltTy = PTy->getElementType(); - - // Figure out if there is a Packed type corresponding to this Vector - // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems); - if (TVT != MVT::Other && isTypeLegal(TVT)) { - // Insert a VBIT_CONVERT of the MVT::Vector type to the vector type. - Op = DAG.getNode(ISD::VBIT_CONVERT, TVT, Op); - Ops.push_back(Op); - Ops.push_back(DAG.getConstant(Flags, MVT::i32)); - } else { - assert(0 && "Don't support illegal by-val vector call args yet!"); - abort(); - } + case Expand: { + MVT::ValueType PartVT = getRegisterType(VT); + unsigned NumParts = getNumRegisters(VT); + SmallVector Parts(NumParts); + getCopyToParts(DAG, Op, &Parts[0], NumParts, PartVT); + for (unsigned i = 0; i != NumParts; ++i) { + // if it isn't first piece, alignment must be 1 + unsigned MyFlags = Flags; + if (i != 0) + MyFlags = (MyFlags & (~ISD::ParamFlags::OrigAlignment)) | + (1 << ISD::ParamFlags::OrigAlignmentOffs); + + Ops.push_back(Parts[i]); + Ops.push_back(DAG.getConstant(MyFlags, MVT::i32)); } break; } + } } // Figure out the result value types. - SmallVector RetTys; - - if (RetTy != Type::VoidTy) { - MVT::ValueType VT = getValueType(RetTy); - switch (getTypeAction(VT)) { - default: assert(0 && "Unknown type action!"); - case Legal: - RetTys.push_back(VT); - break; - case Promote: - RetTys.push_back(getTypeToTransformTo(VT)); - break; - case Expand: - if (VT != MVT::Vector) { - // If this is a large integer, it needs to be reassembled from small - // integers. Figure out what the source elt type is and how many small - // integers it is. - MVT::ValueType NVT = getTypeToExpandTo(VT); - unsigned NumVals = getNumRegisters(VT); - for (unsigned i = 0; i != NumVals; ++i) - RetTys.push_back(NVT); - } else { - // Otherwise, this is a vector type. We only support legal vectors - // right now. - const VectorType *PTy = cast(RetTy); - unsigned NumElems = PTy->getNumElements(); - const Type *EltTy = PTy->getElementType(); - - // Figure out if there is a Packed type corresponding to this Vector - // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems); - if (TVT != MVT::Other && isTypeLegal(TVT)) { - RetTys.push_back(TVT); - } else { - assert(0 && "Don't support illegal by-val vector call results yet!"); - abort(); - } - } - } - } + MVT::ValueType VT = getValueType(RetTy); + MVT::ValueType RegisterVT = getRegisterType(VT); + unsigned NumRegs = getNumRegisters(VT); + SmallVector RetTys(NumRegs); + for (unsigned i = 0; i != NumRegs; ++i) + RetTys[i] = RegisterVT; RetTys.push_back(MVT::Other); // Always has a chain. - // Finally, create the CALL node. + // Create the CALL node. SDOperand Res = DAG.getNode(ISD::CALL, - DAG.getVTList(&RetTys[0], RetTys.size()), + DAG.getVTList(&RetTys[0], NumRegs + 1), &Ops[0], Ops.size()); - - // This returns a pair of operands. The first element is the - // return value for the function (if RetTy is not VoidTy). The second - // element is the outgoing token chain. - SDOperand ResVal; - if (RetTys.size() != 1) { - MVT::ValueType VT = getValueType(RetTy); - if (RetTys.size() == 2) { - ResVal = Res; - - // If this value was promoted, truncate it down. - if (ResVal.getValueType() != VT) { - if (VT == MVT::Vector) { - // Insert a VBIT_CONVERT to convert from the packed result type to the - // MVT::Vector type. - unsigned NumElems = cast(RetTy)->getNumElements(); - const Type *EltTy = cast(RetTy)->getElementType(); - - // Figure out if there is a Packed type corresponding to this Vector - // type. If so, convert to the vector type. - MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy),NumElems); - if (TVT != MVT::Other && isTypeLegal(TVT)) { - // Insert a VBIT_CONVERT of the FORMAL_ARGUMENTS to a - // "N x PTyElementVT" MVT::Vector type. - ResVal = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, ResVal, - DAG.getConstant(NumElems, MVT::i32), - DAG.getValueType(getValueType(EltTy))); - } else { - abort(); - } - } else if (MVT::isInteger(VT)) { - unsigned AssertOp = ISD::AssertSext; - if (!RetTyIsSigned) - AssertOp = ISD::AssertZext; - ResVal = DAG.getNode(AssertOp, ResVal.getValueType(), ResVal, - DAG.getValueType(VT)); - ResVal = DAG.getNode(ISD::TRUNCATE, VT, ResVal); - } else { - assert(MVT::isFloatingPoint(VT)); - if (getTypeAction(VT) == Expand) - ResVal = DAG.getNode(ISD::BIT_CONVERT, VT, ResVal); - else - ResVal = DAG.getNode(ISD::FP_ROUND, VT, ResVal); - } - } - } else if (RetTys.size() == 3) { - ResVal = DAG.getNode(ISD::BUILD_PAIR, VT, - Res.getValue(0), Res.getValue(1)); - - } else { - assert(0 && "Case not handled yet!"); - } + Chain = Res.getValue(NumRegs); + + // Gather up the call result into a single value. + if (RetTy != Type::VoidTy) { + ISD::NodeType AssertOp = ISD::AssertSext; + if (!RetTyIsSigned) + AssertOp = ISD::AssertZext; + SmallVector Results(NumRegs); + for (unsigned i = 0; i != NumRegs; ++i) + Results[i] = Res.getValue(i); + Res = getCopyFromParts(DAG, &Results[0], NumRegs, RegisterVT, VT, AssertOp); } - - return std::make_pair(ResVal, Res.getValue(Res.Val->getNumValues()-1)); + + return std::make_pair(Res, Chain); } SDOperand TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { @@ -4302,6 +4257,17 @@ void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) { unsigned Align = (unsigned)cast(Op4)->getValue(); if (Align == 0) Align = 1; + // If the source and destination are known to not be aliases, we can + // lower memmove as memcpy. + if (Op == ISD::MEMMOVE) { + uint64_t Size = -1ULL; + if (ConstantSDNode *C = dyn_cast(Op3)) + Size = C->getValue(); + if (AA.alias(I.getOperand(1), Size, I.getOperand(2), Size) == + AliasAnalysis::NoAlias) + Op = ISD::MEMCPY; + } + if (ConstantSDNode *Size = dyn_cast(Op3)) { std::vector MemOps; @@ -4370,13 +4336,13 @@ void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) { I.getOperand(1), DstOff); } else { Value = DAG.getLoad(VT, getRoot(), - getMemBasePlusOffset(Op2, SrcOff, DAG, TLI), - I.getOperand(2), SrcOff); + getMemBasePlusOffset(Op2, SrcOff, DAG, TLI), + I.getOperand(2), SrcOff, false, Align); Chain = Value.getValue(1); Store = DAG.getStore(Chain, Value, getMemBasePlusOffset(Op1, DstOff, DAG, TLI), - I.getOperand(1), DstOff); + I.getOperand(1), DstOff, false, Align); } OutChains.push_back(Store); SrcOff += VTSize; @@ -4394,7 +4360,22 @@ void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) { } } - DAG.setRoot(DAG.getNode(Op, MVT::Other, getRoot(), Op1, Op2, Op3, Op4)); + SDOperand AlwaysInline = DAG.getConstant(0, MVT::i1); + SDOperand Node; + switch(Op) { + default: + assert(0 && "Unknown Op"); + case ISD::MEMCPY: + Node = DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline); + break; + case ISD::MEMMOVE: + Node = DAG.getMemmove(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline); + break; + case ISD::MEMSET: + Node = DAG.getMemset(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline); + break; + } + DAG.setRoot(Node); } //===----------------------------------------------------------------------===// @@ -4402,19 +4383,27 @@ void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) { //===----------------------------------------------------------------------===// unsigned SelectionDAGISel::MakeReg(MVT::ValueType VT) { - return RegMap->createVirtualRegister(TLI.getRegClassFor(VT)); + return RegInfo->createVirtualRegister(TLI.getRegClassFor(VT)); } void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); + AU.addRequired(); AU.setPreservesAll(); } bool SelectionDAGISel::runOnFunction(Function &Fn) { + // Get alias analysis for load/store combining. + AA = &getAnalysis(); + MachineFunction &MF = MachineFunction::construct(&Fn, TLI.getTargetMachine()); - RegMap = MF.getSSARegMap(); + if (MF.getFunction()->hasCollector()) + GCI = &getAnalysis().get(*MF.getFunction()); + else + GCI = 0; + RegInfo = &MF.getRegInfo(); DOUT << "\n\n\n=== " << Fn.getName() << "\n"; FunctionLoweringInfo FuncInfo(TLI, Fn, MF); @@ -4431,9 +4420,9 @@ bool SelectionDAGISel::runOnFunction(Function &Fn) { // Add function live-ins to entry block live-in set. BasicBlock *EntryBB = &Fn.getEntryBlock(); BB = FuncInfo.MBBMap[EntryBB]; - if (!MF.livein_empty()) - for (MachineFunction::livein_iterator I = MF.livein_begin(), - E = MF.livein_end(); I != E; ++I) + if (!RegInfo->livein_empty()) + for (MachineRegisterInfo::livein_iterator I = RegInfo->livein_begin(), + E = RegInfo->livein_end(); I != E; ++I) BB->addLiveIn(I->first); #ifndef NDEBUG @@ -4451,75 +4440,17 @@ SDOperand SelectionDAGLowering::CopyValueToVirtualRegister(Value *V, cast(Op.getOperand(1))->getReg() != Reg) && "Copy from a reg to the same reg!"); - // If this type is not legal, we must make sure to not create an invalid - // register use. MVT::ValueType SrcVT = Op.getValueType(); - MVT::ValueType DestVT = TLI.getTypeToTransformTo(SrcVT); - if (SrcVT == DestVT) { - return DAG.getCopyToReg(getRoot(), Reg, Op); - } else if (SrcVT == MVT::Vector) { - // Handle copies from generic vectors to registers. - MVT::ValueType PTyElementVT, PTyLegalElementVT; - unsigned NE = TLI.getVectorTypeBreakdown(cast(V->getType()), - PTyElementVT, PTyLegalElementVT); - uint64_t SrcVL = cast(*(Op.Val->op_end()-2))->getValue(); - - // Loop over all of the elements of the resultant vector, - // VEXTRACT_VECTOR_ELT'ing or VEXTRACT_SUBVECTOR'ing them, converting them - // to PTyLegalElementVT, then copying them into output registers. - SmallVector OutChains; - SDOperand Root = getRoot(); - for (unsigned i = 0; i != NE; ++i) { - SDOperand Elt = MVT::isVector(PTyElementVT) ? - DAG.getNode(ISD::VEXTRACT_SUBVECTOR, PTyElementVT, - Op, DAG.getConstant(i * (SrcVL / NE), TLI.getPointerTy())) : - DAG.getNode(ISD::VEXTRACT_VECTOR_ELT, PTyElementVT, - Op, DAG.getConstant(i, TLI.getPointerTy())); - if (PTyElementVT == PTyLegalElementVT) { - // Elements are legal. - OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Elt)); - } else if (PTyLegalElementVT > PTyElementVT) { - // Elements are promoted. - if (MVT::isFloatingPoint(PTyLegalElementVT)) - Elt = DAG.getNode(ISD::FP_EXTEND, PTyLegalElementVT, Elt); - else - Elt = DAG.getNode(ISD::ANY_EXTEND, PTyLegalElementVT, Elt); - OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Elt)); - } else { - // Elements are expanded. - // The src value is expanded into multiple registers. - SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, PTyLegalElementVT, - Elt, DAG.getConstant(0, TLI.getPointerTy())); - SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, PTyLegalElementVT, - Elt, DAG.getConstant(1, TLI.getPointerTy())); - OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Lo)); - OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Hi)); - } - } - return DAG.getNode(ISD::TokenFactor, MVT::Other, - &OutChains[0], OutChains.size()); - } else if (TLI.getTypeAction(SrcVT) == TargetLowering::Promote) { - // The src value is promoted to the register. - if (MVT::isFloatingPoint(SrcVT)) - Op = DAG.getNode(ISD::FP_EXTEND, DestVT, Op); - else - Op = DAG.getNode(ISD::ANY_EXTEND, DestVT, Op); - return DAG.getCopyToReg(getRoot(), Reg, Op); - } else { - DestVT = TLI.getTypeToExpandTo(SrcVT); - unsigned NumVals = TLI.getNumRegisters(SrcVT); - if (NumVals == 1) - return DAG.getCopyToReg(getRoot(), Reg, - DAG.getNode(ISD::BIT_CONVERT, DestVT, Op)); - assert(NumVals == 2 && "1 to 4 (and more) expansion not implemented!"); - // The src value is expanded into multiple registers. - SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DestVT, - Op, DAG.getConstant(0, TLI.getPointerTy())); - SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DestVT, - Op, DAG.getConstant(1, TLI.getPointerTy())); - Op = DAG.getCopyToReg(getRoot(), Reg, Lo); - return DAG.getCopyToReg(Op, Reg+1, Hi); - } + MVT::ValueType RegisterVT = TLI.getRegisterType(SrcVT); + unsigned NumRegs = TLI.getNumRegisters(SrcVT); + SmallVector Regs(NumRegs); + SmallVector Chains(NumRegs); + + // Copy the value by legal parts into sequential virtual registers. + getCopyToParts(DAG, Op, &Regs[0], NumRegs, RegisterVT); + for (unsigned i = 0; i != NumRegs; ++i) + Chains[i] = DAG.getCopyToReg(getRoot(), Reg + i, Regs[i]); + return DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumRegs); } void SelectionDAGISel:: @@ -4553,22 +4484,63 @@ LowerArguments(BasicBlock *LLVMBB, SelectionDAGLowering &SDL, static void copyCatchInfo(BasicBlock *SrcBB, BasicBlock *DestBB, MachineModuleInfo *MMI, FunctionLoweringInfo &FLI) { - assert(!FLI.MBBMap[SrcBB]->isLandingPad() && - "Copying catch info out of a landing pad!"); for (BasicBlock::iterator I = SrcBB->begin(), E = --SrcBB->end(); I != E; ++I) - if (isFilterOrSelector(I)) { + if (isSelector(I)) { // Apply the catch info to DestBB. addCatchInfo(cast(*I), MMI, FLI.MBBMap[DestBB]); #ifndef NDEBUG - FLI.CatchInfoFound.insert(I); + if (!FLI.MBBMap[SrcBB]->isLandingPad()) + FLI.CatchInfoFound.insert(I); #endif } } +/// CheckDAGForTailCallsAndFixThem - This Function looks for CALL nodes in the +/// DAG and fixes their tailcall attribute operand. +static void CheckDAGForTailCallsAndFixThem(SelectionDAG &DAG, + TargetLowering& TLI) { + SDNode * Ret = NULL; + SDOperand Terminator = DAG.getRoot(); + + // Find RET node. + if (Terminator.getOpcode() == ISD::RET) { + Ret = Terminator.Val; + } + + // Fix tail call attribute of CALL nodes. + for (SelectionDAG::allnodes_iterator BE = DAG.allnodes_begin(), + BI = prior(DAG.allnodes_end()); BI != BE; --BI) { + if (BI->getOpcode() == ISD::CALL) { + SDOperand OpRet(Ret, 0); + SDOperand OpCall(static_cast(BI), 0); + bool isMarkedTailCall = + cast(OpCall.getOperand(3))->getValue() != 0; + // If CALL node has tail call attribute set to true and the call is not + // eligible (no RET or the target rejects) the attribute is fixed to + // false. The TargetLowering::IsEligibleForTailCallOptimization function + // must correctly identify tail call optimizable calls. + if (isMarkedTailCall && + (Ret==NULL || + !TLI.IsEligibleForTailCallOptimization(OpCall, OpRet, DAG))) { + SmallVector Ops; + unsigned idx=0; + for(SDNode::op_iterator I =OpCall.Val->op_begin(), + E=OpCall.Val->op_end(); I!=E; I++, idx++) { + if (idx!=3) + Ops.push_back(*I); + else + Ops.push_back(DAG.getConstant(false, TLI.getPointerTy())); + } + DAG.UpdateNodeOperands(OpCall, Ops.begin(), Ops.size()); + } + } + } +} + void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, std::vector > &PHINodesToUpdate, FunctionLoweringInfo &FuncInfo) { - SelectionDAGLowering SDL(DAG, TLI, FuncInfo); + SelectionDAGLowering SDL(DAG, TLI, *AA, FuncInfo, GCI); std::vector UnorderedChains; @@ -4588,23 +4560,31 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, DAG.getEntryNode(), DAG.getConstant(LabelID, MVT::i32))); + // Mark exception register as live in. + unsigned Reg = TLI.getExceptionAddressRegister(); + if (Reg) BB->addLiveIn(Reg); + + // Mark exception selector register as live in. + Reg = TLI.getExceptionSelectorRegister(); + if (Reg) BB->addLiveIn(Reg); + // FIXME: Hack around an exception handling flaw (PR1508): the personality // function and list of typeids logically belong to the invoke (or, if you // like, the basic block containing the invoke), and need to be associated // with it in the dwarf exception handling tables. Currently however the - // information is provided by intrinsics (eh.filter and eh.selector) that - // can be moved to unexpected places by the optimizers: if the unwind edge - // is critical, then breaking it can result in the intrinsics being in the - // successor of the landing pad, not the landing pad itself. This results - // in exceptions not being caught because no typeids are associated with - // the invoke. This may not be the only way things can go wrong, but it - // is the only way we try to work around for the moment. + // information is provided by an intrinsic (eh.selector) that can be moved + // to unexpected places by the optimizers: if the unwind edge is critical, + // then breaking it can result in the intrinsics being in the successor of + // the landing pad, not the landing pad itself. This results in exceptions + // not being caught because no typeids are associated with the invoke. + // This may not be the only way things can go wrong, but it is the only way + // we try to work around for the moment. BranchInst *Br = dyn_cast(LLVMBB->getTerminator()); if (Br && Br->isUnconditional()) { // Critical edge? BasicBlock::iterator I, E; for (I = LLVMBB->begin(), E = --LLVMBB->end(); I != E; ++I) - if (isFilterOrSelector(I)) + if (isSelector(I)) break; if (I == E) @@ -4645,8 +4625,8 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, if (TI->getNumSuccessors()) SuccsHandled.resize(BB->getParent()->getNumBlockIDs()); - // Check successor nodes PHI nodes that expect a constant to be available from - // this block. + // Check successor nodes' PHI nodes that expect a constant to be available + // from this block. for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) { BasicBlock *SuccBB = TI->getSuccessor(succ); if (!isa(SuccBB->begin())) continue; @@ -4695,15 +4675,7 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, // Remember that this register needs to added to the machine PHI node as // the input for this MBB. MVT::ValueType VT = TLI.getValueType(PN->getType()); - unsigned NumRegisters; - if (VT != MVT::Vector) - NumRegisters = TLI.getNumRegisters(VT); - else { - MVT::ValueType VT1,VT2; - NumRegisters = - TLI.getVectorTypeBreakdown(cast(PN->getType()), - VT1, VT2); - } + unsigned NumRegisters = TLI.getNumRegisters(VT); for (unsigned i = 0, e = NumRegisters; i != e; ++i) PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); } @@ -4742,28 +4714,41 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, // Make sure the root of the DAG is up-to-date. DAG.setRoot(SDL.getRoot()); + + // Check whether calls in this block are real tail calls. Fix up CALL nodes + // with correct tailcall attribute so that the target can rely on the tailcall + // attribute indicating whether the call is really eligible for tail call + // optimization. + CheckDAGForTailCallsAndFixThem(DAG, TLI); } void SelectionDAGISel::CodeGenAndEmitDAG(SelectionDAG &DAG) { - // Get alias analysis for load/store combining. - AliasAnalysis &AA = getAnalysis(); + DOUT << "Lowered selection DAG:\n"; + DEBUG(DAG.dump()); // Run the DAG combiner in pre-legalize mode. - DAG.Combine(false, AA); + DAG.Combine(false, *AA); - DOUT << "Lowered selection DAG:\n"; + DOUT << "Optimized lowered selection DAG:\n"; DEBUG(DAG.dump()); // Second step, hack on the DAG until it only uses operations and types that // the target supports. +#if 0 // Enable this some day. + DAG.LegalizeTypes(); + // Someday even later, enable a dag combine pass here. +#endif DAG.Legalize(); DOUT << "Legalized selection DAG:\n"; DEBUG(DAG.dump()); // Run the DAG combiner in post-legalize mode. - DAG.Combine(true, AA); + DAG.Combine(true, *AA); + DOUT << "Optimized legalized selection DAG:\n"; + DEBUG(DAG.dump()); + if (ViewISelDAGs) DAG.viewGraph(); // Third, instruction select all of the operations to machine code, adding the @@ -4802,8 +4787,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, MachineInstr *PHI = PHINodesToUpdate[i].first; assert(PHI->getOpcode() == TargetInstrInfo::PHI && "This is not a machine PHI node that we are updating!"); - PHI->addRegOperand(PHINodesToUpdate[i].second, false); - PHI->addMachineBasicBlockOperand(BB); + PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[i].second, + false)); + PHI->addOperand(MachineOperand::CreateMBB(BB)); } return; } @@ -4813,7 +4799,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, if (!BitTestCases[i].Emitted) { SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate()); CurDAG = &HSDAG; - SelectionDAGLowering HSDL(HSDAG, TLI, FuncInfo); + SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into BB = BitTestCases[i].Parent; HSDL.setCurrentBasicBlock(BB); @@ -4826,7 +4812,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, for (unsigned j = 0, ej = BitTestCases[i].Cases.size(); j != ej; ++j) { SelectionDAG BSDAG(TLI, MF, getAnalysisToUpdate()); CurDAG = &BSDAG; - SelectionDAGLowering BSDL(BSDAG, TLI, FuncInfo); + SelectionDAGLowering BSDL(BSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into BB = BitTestCases[i].Cases[j].ThisBB; BSDL.setCurrentBasicBlock(BB); @@ -4854,18 +4840,22 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, // This is "default" BB. We have two jumps to it. From "header" BB and // from last "case" BB. if (PHIBB == BitTestCases[i].Default) { - PHI->addRegOperand(PHINodesToUpdate[pi].second, false); - PHI->addMachineBasicBlockOperand(BitTestCases[i].Parent); - PHI->addRegOperand(PHINodesToUpdate[pi].second, false); - PHI->addMachineBasicBlockOperand(BitTestCases[i].Cases.back().ThisBB); + PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pi].second, + false)); + PHI->addOperand(MachineOperand::CreateMBB(BitTestCases[i].Parent)); + PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pi].second, + false)); + PHI->addOperand(MachineOperand::CreateMBB(BitTestCases[i].Cases. + back().ThisBB)); } // One of "cases" BB. for (unsigned j = 0, ej = BitTestCases[i].Cases.size(); j != ej; ++j) { MachineBasicBlock* cBB = BitTestCases[i].Cases[j].ThisBB; if (cBB->succ_end() != std::find(cBB->succ_begin(),cBB->succ_end(), PHIBB)) { - PHI->addRegOperand(PHINodesToUpdate[pi].second, false); - PHI->addMachineBasicBlockOperand(cBB); + PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pi].second, + false)); + PHI->addOperand(MachineOperand::CreateMBB(cBB)); } } } @@ -4879,7 +4869,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, if (!JTCases[i].first.Emitted) { SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate()); CurDAG = &HSDAG; - SelectionDAGLowering HSDL(HSDAG, TLI, FuncInfo); + SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into BB = JTCases[i].first.HeaderBB; HSDL.setCurrentBasicBlock(BB); @@ -4891,7 +4881,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, SelectionDAG JSDAG(TLI, MF, getAnalysisToUpdate()); CurDAG = &JSDAG; - SelectionDAGLowering JSDL(JSDAG, TLI, FuncInfo); + SelectionDAGLowering JSDL(JSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into BB = JTCases[i].second.MBB; JSDL.setCurrentBasicBlock(BB); @@ -4908,13 +4898,15 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, "This is not a machine PHI node that we are updating!"); // "default" BB. We can go there only from header BB. if (PHIBB == JTCases[i].second.Default) { - PHI->addRegOperand(PHINodesToUpdate[pi].second, false); - PHI->addMachineBasicBlockOperand(JTCases[i].first.HeaderBB); + PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pi].second, + false)); + PHI->addOperand(MachineOperand::CreateMBB(JTCases[i].first.HeaderBB)); } // JT BB. Just iterate over successors here if (BB->succ_end() != std::find(BB->succ_begin(),BB->succ_end(), PHIBB)) { - PHI->addRegOperand(PHINodesToUpdate[pi].second, false); - PHI->addMachineBasicBlockOperand(BB); + PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pi].second, + false)); + PHI->addOperand(MachineOperand::CreateMBB(BB)); } } } @@ -4926,8 +4918,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, assert(PHI->getOpcode() == TargetInstrInfo::PHI && "This is not a machine PHI node that we are updating!"); if (BB->isSuccessor(PHI->getParent())) { - PHI->addRegOperand(PHINodesToUpdate[i].second, false); - PHI->addMachineBasicBlockOperand(BB); + PHI->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[i].second, + false)); + PHI->addOperand(MachineOperand::CreateMBB(BB)); } } @@ -4936,7 +4929,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, for (unsigned i = 0, e = SwitchCases.size(); i != e; ++i) { SelectionDAG SDAG(TLI, MF, getAnalysisToUpdate()); CurDAG = &SDAG; - SelectionDAGLowering SDL(SDAG, TLI, FuncInfo); + SelectionDAGLowering SDL(SDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into BB = SwitchCases[i].ThisBB; @@ -4958,8 +4951,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, for (unsigned pn = 0; ; ++pn) { assert(pn != PHINodesToUpdate.size() && "Didn't find PHI entry!"); if (PHINodesToUpdate[pn].first == Phi) { - Phi->addRegOperand(PHINodesToUpdate[pn].second, false); - Phi->addMachineBasicBlockOperand(SwitchCases[i].ThisBB); + Phi->addOperand(MachineOperand::CreateReg(PHINodesToUpdate[pn]. + second, false)); + Phi->addOperand(MachineOperand::CreateMBB(SwitchCases[i].ThisBB)); break; } } @@ -4993,6 +4987,9 @@ void SelectionDAGISel::ScheduleAndEmitDAG(SelectionDAG &DAG) { ScheduleDAG *SL = Ctor(this, &DAG, BB); BB = SL->Run(); + + if (ViewSUnitDAGs) SL->viewGraph(); + delete SL; } @@ -5011,7 +5008,7 @@ HazardRecognizer *SelectionDAGISel::CreateTargetHazardRecognizer() { /// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value /// specified in the .td file (e.g. 255). bool SelectionDAGISel::CheckAndMask(SDOperand LHS, ConstantSDNode *RHS, - int64_t DesiredMaskS) { + int64_t DesiredMaskS) const { uint64_t ActualMask = RHS->getValue(); uint64_t DesiredMask =DesiredMaskS & MVT::getIntVTBitMask(LHS.getValueType()); @@ -5040,7 +5037,7 @@ bool SelectionDAGISel::CheckAndMask(SDOperand LHS, ConstantSDNode *RHS, /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value /// specified in the .td file (e.g. 255). bool SelectionDAGISel::CheckOrMask(SDOperand LHS, ConstantSDNode *RHS, - int64_t DesiredMaskS) { + int64_t DesiredMaskS) const { uint64_t ActualMask = RHS->getValue(); uint64_t DesiredMask =DesiredMaskS & MVT::getIntVTBitMask(LHS.getValueType());