X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAGBuilder.cpp;h=db127f260ffd3701d310522514c76940d2474cfc;hb=f031e8ad011e9ad95d7c965936da07e3a9c42add;hp=657ba628927cefdfd76f127afc7cb162047ac090;hpb=87710f04e551655c0c98fb7c0bfd9ec58224a804;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 657ba628927..db127f260ff 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -17,6 +17,7 @@ #include "llvm/ADT/BitVector.h" #include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/ConstantFolding.h" #include "llvm/Constants.h" #include "llvm/CallingConv.h" #include "llvm/DerivedTypes.h" @@ -26,7 +27,6 @@ #include "llvm/Instructions.h" #include "llvm/Intrinsics.h" #include "llvm/IntrinsicInst.h" -#include "llvm/LLVMContext.h" #include "llvm/Module.h" #include "llvm/CodeGen/FastISel.h" #include "llvm/CodeGen/GCStrategy.h" @@ -144,22 +144,23 @@ namespace { /// this value and returns the result as a ValueVTs value. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. - SDValue getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, - SDValue &Chain, SDValue *Flag) const; + SDValue getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, unsigned Order, + SDValue &Chain, SDValue *Flag) const; /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the /// specified value into the registers specified by this object. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. void getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, - SDValue &Chain, SDValue *Flag) const; + unsigned Order, SDValue &Chain, SDValue *Flag) const; /// AddInlineAsmOperands - Add this value to the specified inlineasm node /// operand list. This adds the code marker, matching input operand index /// (if applicable), and includes the number of values added into it. void AddInlineAsmOperands(unsigned Code, bool HasMatching, unsigned MatchingIdx, - SelectionDAG &DAG, std::vector &Ops) const; + SelectionDAG &DAG, unsigned Order, + std::vector &Ops) const; }; } @@ -168,13 +169,14 @@ namespace { /// larger then ValueVT then AssertOp can be used to specify whether the extra /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT /// (ISD::AssertSext). -static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, +static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, unsigned Order, const SDValue *Parts, unsigned NumParts, EVT PartVT, EVT ValueVT, ISD::NodeType AssertOp = ISD::DELETED_NODE) { assert(NumParts > 0 && "No parts to assemble!"); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue Val = Parts[0]; + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); if (NumParts > 1) { // Assemble the value from multiple parts. @@ -193,23 +195,32 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); if (RoundParts > 2) { - Lo = getCopyFromParts(DAG, dl, Parts, RoundParts/2, PartVT, HalfVT); - Hi = getCopyFromParts(DAG, dl, Parts+RoundParts/2, RoundParts/2, + Lo = getCopyFromParts(DAG, dl, Order, Parts, RoundParts / 2, PartVT, HalfVT); + Hi = getCopyFromParts(DAG, dl, Order, Parts + RoundParts / 2, + RoundParts / 2, PartVT, HalfVT); } else { Lo = DAG.getNode(ISD::BIT_CONVERT, dl, HalfVT, Parts[0]); Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HalfVT, Parts[1]); } + if (TLI.isBigEndian()) std::swap(Lo, Hi); + Val = DAG.getNode(ISD::BUILD_PAIR, dl, RoundVT, Lo, Hi); + if (DisableScheduling) { + DAG.AssignOrdering(Lo.getNode(), Order); + DAG.AssignOrdering(Hi.getNode(), Order); + DAG.AssignOrdering(Val.getNode(), Order); + } + if (RoundParts < NumParts) { // Assemble the trailing non-power-of-2 part. unsigned OddParts = NumParts - RoundParts; EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits); - Hi = getCopyFromParts(DAG, dl, - Parts+RoundParts, OddParts, PartVT, OddVT); + Hi = getCopyFromParts(DAG, dl, Order, + Parts + RoundParts, OddParts, PartVT, OddVT); // Combine the round and odd parts. Lo = Val; @@ -217,11 +228,15 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, std::swap(Lo, Hi); EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); Hi = DAG.getNode(ISD::ANY_EXTEND, dl, TotalVT, Hi); + if (DisableScheduling) DAG.AssignOrdering(Hi.getNode(), Order); Hi = DAG.getNode(ISD::SHL, dl, TotalVT, Hi, DAG.getConstant(Lo.getValueType().getSizeInBits(), TLI.getPointerTy())); + if (DisableScheduling) DAG.AssignOrdering(Hi.getNode(), Order); Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, TotalVT, Lo); + if (DisableScheduling) DAG.AssignOrdering(Lo.getNode(), Order); Val = DAG.getNode(ISD::OR, dl, TotalVT, Lo, Hi); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); } } else if (ValueVT.isVector()) { // Handle a multi-element vector. @@ -242,7 +257,7 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, // If the register was not expanded, truncate or copy the value, // as appropriate. for (unsigned i = 0; i != NumParts; ++i) - Ops[i] = getCopyFromParts(DAG, dl, &Parts[i], 1, + Ops[i] = getCopyFromParts(DAG, dl, Order, &Parts[i], 1, PartVT, IntermediateVT); } else if (NumParts > 0) { // If the intermediate type was expanded, build the intermediate operands @@ -251,7 +266,7 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, "Must expand into a divisible number of parts!"); unsigned Factor = NumParts / NumIntermediates; for (unsigned i = 0; i != NumIntermediates; ++i) - Ops[i] = getCopyFromParts(DAG, dl, &Parts[i * Factor], Factor, + Ops[i] = getCopyFromParts(DAG, dl, Order, &Parts[i * Factor], Factor, PartVT, IntermediateVT); } @@ -260,6 +275,7 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR, dl, ValueVT, &Ops[0], NumIntermediates); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); } else if (PartVT.isFloatingPoint()) { // FP split into multiple FP parts (for ppcf128) assert(ValueVT == EVT(MVT::ppcf128) && PartVT == EVT(MVT::f64) && @@ -270,12 +286,18 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, if (TLI.isBigEndian()) std::swap(Lo, Hi); Val = DAG.getNode(ISD::BUILD_PAIR, dl, ValueVT, Lo, Hi); + + if (DisableScheduling) { + DAG.AssignOrdering(Hi.getNode(), Order); + DAG.AssignOrdering(Lo.getNode(), Order); + DAG.AssignOrdering(Val.getNode(), Order); + } } else { // FP split into integer parts (soft fp) assert(ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"); EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); - Val = getCopyFromParts(DAG, dl, Parts, NumParts, PartVT, IntVT); + Val = getCopyFromParts(DAG, dl, Order, Parts, NumParts, PartVT, IntVT); } } @@ -287,14 +309,20 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, if (PartVT.isVector()) { assert(ValueVT.isVector() && "Unknown vector conversion!"); - return DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val); + SDValue Res = DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); + return Res; } if (ValueVT.isVector()) { assert(ValueVT.getVectorElementType() == PartVT && ValueVT.getVectorNumElements() == 1 && "Only trivial scalar-to-vector conversions should get here!"); - return DAG.getNode(ISD::BUILD_VECTOR, dl, ValueVT, Val); + SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, dl, ValueVT, Val); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); + return Res; } if (PartVT.isInteger() && @@ -306,22 +334,36 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, if (AssertOp != ISD::DELETED_NODE) Val = DAG.getNode(AssertOp, dl, PartVT, Val, DAG.getValueType(ValueVT)); - return DAG.getNode(ISD::TRUNCATE, dl, ValueVT, Val); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + Val = DAG.getNode(ISD::TRUNCATE, dl, ValueVT, Val); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + return Val; } else { - return DAG.getNode(ISD::ANY_EXTEND, dl, ValueVT, Val); + Val = DAG.getNode(ISD::ANY_EXTEND, dl, ValueVT, Val); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + return Val; } } if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { - if (ValueVT.bitsLT(Val.getValueType())) + if (ValueVT.bitsLT(Val.getValueType())) { // FP_ROUND's are always exact here. - return DAG.getNode(ISD::FP_ROUND, dl, ValueVT, Val, - DAG.getIntPtrConstant(1)); - return DAG.getNode(ISD::FP_EXTEND, dl, ValueVT, Val); + Val = DAG.getNode(ISD::FP_ROUND, dl, ValueVT, Val, + DAG.getIntPtrConstant(1)); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + return Val; + } + + Val = DAG.getNode(ISD::FP_EXTEND, dl, ValueVT, Val); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + return Val; } - if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) - return DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val); + if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { + Val = DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + return Val; + } llvm_unreachable("Unknown mismatch!"); return SDValue(); @@ -330,8 +372,9 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, /// getCopyToParts - Create a series of nodes that contain the specified value /// split into legal parts. If the parts contain more bits than Val, then, for /// integers, ExtendKind can be used to specify how to generate the extra bits. -static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, - SDValue *Parts, unsigned NumParts, EVT PartVT, +static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, unsigned Order, + SDValue Val, SDValue *Parts, unsigned NumParts, + EVT PartVT, ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT PtrVT = TLI.getPointerTy(); @@ -375,6 +418,8 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, } } + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + // The value may have changed - recompute ValueVT. ValueVT = Val.getValueType(); assert(NumParts * PartBits == ValueVT.getSizeInBits() && @@ -397,20 +442,33 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, SDValue OddVal = DAG.getNode(ISD::SRL, dl, ValueVT, Val, DAG.getConstant(RoundBits, TLI.getPointerTy())); - getCopyToParts(DAG, dl, OddVal, Parts + RoundParts, OddParts, PartVT); + getCopyToParts(DAG, dl, Order, OddVal, Parts + RoundParts, + OddParts, PartVT); + if (TLI.isBigEndian()) // The odd parts were reversed by getCopyToParts - unreverse them. std::reverse(Parts + RoundParts, Parts + NumParts); + NumParts = RoundParts; ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); Val = DAG.getNode(ISD::TRUNCATE, dl, ValueVT, Val); + + if (DisableScheduling) { + DAG.AssignOrdering(OddVal.getNode(), Order); + DAG.AssignOrdering(Val.getNode(), Order); + } } // The number of parts is a power of 2. Repeatedly bisect the value using // EXTRACT_ELEMENT. Parts[0] = DAG.getNode(ISD::BIT_CONVERT, dl, - EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()), + EVT::getIntegerVT(*DAG.getContext(), + ValueVT.getSizeInBits()), Val); + + if (DisableScheduling) + DAG.AssignOrdering(Parts[0].getNode(), Order); + for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { for (unsigned i = 0; i < NumParts; i += StepSize) { unsigned ThisBits = StepSize * PartBits / 2; @@ -425,11 +483,20 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, ThisVT, Part0, DAG.getConstant(0, PtrVT)); + if (DisableScheduling) { + DAG.AssignOrdering(Part0.getNode(), Order); + DAG.AssignOrdering(Part1.getNode(), Order); + } + if (ThisBits == PartBits && ThisVT != PartVT) { Part0 = DAG.getNode(ISD::BIT_CONVERT, dl, PartVT, Part0); Part1 = DAG.getNode(ISD::BIT_CONVERT, dl, PartVT, Part1); + if (DisableScheduling) { + DAG.AssignOrdering(Part0.getNode(), Order); + DAG.AssignOrdering(Part1.getNode(), Order); + } } } } @@ -455,6 +522,9 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, } } + if (DisableScheduling) + DAG.AssignOrdering(Val.getNode(), Order); + Parts[0] = Val; return; } @@ -472,7 +542,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, // Split the vector into intermediate operands. SmallVector Ops(NumIntermediates); - for (unsigned i = 0; i != NumIntermediates; ++i) + for (unsigned i = 0; i != NumIntermediates; ++i) { if (IntermediateVT.isVector()) Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, IntermediateVT, Val, @@ -483,12 +553,16 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, IntermediateVT, Val, DAG.getConstant(i, PtrVT)); + if (DisableScheduling) + DAG.AssignOrdering(Ops[i].getNode(), Order); + } + // Split the intermediate operands into legal parts. if (NumParts == NumIntermediates) { // If the register was not expanded, promote or copy the value, // as appropriate. for (unsigned i = 0; i != NumParts; ++i) - getCopyToParts(DAG, dl, Ops[i], &Parts[i], 1, PartVT); + getCopyToParts(DAG, dl, Order, Ops[i], &Parts[i], 1, PartVT); } else if (NumParts > 0) { // If the intermediate type was expanded, split each the value into // legal parts. @@ -496,7 +570,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, "Must expand into a divisible number of parts!"); unsigned Factor = NumParts / NumIntermediates; for (unsigned i = 0; i != NumIntermediates; ++i) - getCopyToParts(DAG, dl, Ops[i], &Parts[i * Factor], Factor, PartVT); + getCopyToParts(DAG, dl, Order, Ops[i], &Parts[i*Factor], Factor, PartVT); } } @@ -720,10 +794,8 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { RegsForValue RFV(*DAG.getContext(), TLI, InReg, V->getType()); SDValue Chain = DAG.getEntryNode(); - SDValue Res = RFV.getCopyFromRegs(DAG, getCurDebugLoc(), Chain, NULL); - if (DisableScheduling) - DAG.AssignOrdering(Res.getNode(), SDNodeOrder); - return Res; + return RFV.getCopyFromRegs(DAG, getCurDebugLoc(), + SDNodeOrder, Chain, NULL); } /// Get the EVTs and ArgFlags collections that represent the return type @@ -855,7 +927,7 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) { unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), VT); EVT PartVT = TLI.getRegisterType(*DAG.getContext(), VT); SmallVector Parts(NumParts); - getCopyToParts(DAG, getCurDebugLoc(), + getCopyToParts(DAG, getCurDebugLoc(), SDNodeOrder, SDValue(RetOp.getNode(), RetOp.getResNo() + j), &Parts[0], NumParts, PartVT, ExtendKind); @@ -2861,11 +2933,15 @@ void SelectionDAGBuilder::visitAlloca(AllocaInst &I) { AllocSize, DAG.getConstant(TySize, AllocSize.getValueType())); - + if (DisableScheduling) + DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder); EVT IntPtr = TLI.getPointerTy(); AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurDebugLoc(), IntPtr); + if (DisableScheduling) + DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder); + // Handle alignment. If the requested alignment is less than or equal to // the stack alignment, ignore it. If the size is greater than or equal to // the stack alignment, we note this in the DYNAMIC_STACKALLOC node. @@ -2879,10 +2955,15 @@ void SelectionDAGBuilder::visitAlloca(AllocaInst &I) { AllocSize = DAG.getNode(ISD::ADD, getCurDebugLoc(), AllocSize.getValueType(), AllocSize, DAG.getIntPtrConstant(StackAlign-1)); + if (DisableScheduling) + DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder); + // Mask out the low bits for alignment purposes. AllocSize = DAG.getNode(ISD::AND, getCurDebugLoc(), AllocSize.getValueType(), AllocSize, DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1))); + if (DisableScheduling) + DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder); SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) }; SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other); @@ -2891,6 +2972,9 @@ void SelectionDAGBuilder::visitAlloca(AllocaInst &I) { setValue(&I, DSA); DAG.setRoot(DSA.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(DSA.getNode(), SDNodeOrder); + // Inform the Frame Information that we have just allocated a variable-sized // object. FuncInfo.MF->getFrameInfo()->CreateVariableSizedObject(); @@ -2929,30 +3013,41 @@ void SelectionDAGBuilder::visitLoad(LoadInst &I) { SmallVector Chains(NumValues); EVT PtrVT = Ptr.getValueType(); for (unsigned i = 0; i != NumValues; ++i) { + SDValue A = DAG.getNode(ISD::ADD, getCurDebugLoc(), + PtrVT, Ptr, + DAG.getConstant(Offsets[i], PtrVT)); SDValue L = DAG.getLoad(ValueVTs[i], getCurDebugLoc(), Root, - DAG.getNode(ISD::ADD, getCurDebugLoc(), - PtrVT, Ptr, - DAG.getConstant(Offsets[i], PtrVT)), - SV, Offsets[i], isVolatile, Alignment); + A, SV, Offsets[i], isVolatile, Alignment); + Values[i] = L; Chains[i] = L.getValue(1); + + if (DisableScheduling) { + DAG.AssignOrdering(A.getNode(), SDNodeOrder); + DAG.AssignOrdering(L.getNode(), SDNodeOrder); + } } if (!ConstantMemory) { SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), - MVT::Other, - &Chains[0], NumValues); + MVT::Other, &Chains[0], NumValues); if (isVolatile) DAG.setRoot(Chain); else PendingLoads.push_back(Chain); + + if (DisableScheduling) + DAG.AssignOrdering(Chain.getNode(), SDNodeOrder); } - setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(), - DAG.getVTList(&ValueVTs[0], NumValues), - &Values[0], NumValues)); -} + SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(), + DAG.getVTList(&ValueVTs[0], NumValues), + &Values[0], NumValues); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); +} void SelectionDAGBuilder::visitStore(StoreInst &I) { Value *SrcV = I.getOperand(0); @@ -2976,16 +3071,26 @@ void SelectionDAGBuilder::visitStore(StoreInst &I) { EVT PtrVT = Ptr.getValueType(); bool isVolatile = I.isVolatile(); unsigned Alignment = I.getAlignment(); - for (unsigned i = 0; i != NumValues; ++i) + + for (unsigned i = 0; i != NumValues; ++i) { + SDValue Add = DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, Ptr, + DAG.getConstant(Offsets[i], PtrVT)); Chains[i] = DAG.getStore(Root, getCurDebugLoc(), SDValue(Src.getNode(), Src.getResNo() + i), - DAG.getNode(ISD::ADD, getCurDebugLoc(), - PtrVT, Ptr, - DAG.getConstant(Offsets[i], PtrVT)), - PtrV, Offsets[i], isVolatile, Alignment); + Add, PtrV, Offsets[i], isVolatile, Alignment); - DAG.setRoot(DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), - MVT::Other, &Chains[0], NumValues)); + if (DisableScheduling) { + DAG.AssignOrdering(Add.getNode(), SDNodeOrder); + DAG.AssignOrdering(Chains[i].getNode(), SDNodeOrder); + } + } + + SDValue Res = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), + MVT::Other, &Chains[0], NumValues); + DAG.setRoot(Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC @@ -3030,6 +3135,7 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I, "Intrinsic uses a non-legal type?"); } #endif // NDEBUG + if (HasChain) ValueVTs.push_back(MVT::Other); @@ -3044,16 +3150,19 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I, Info.memVT, Info.ptrVal, Info.offset, Info.align, Info.vol, Info.readMem, Info.writeMem); - } - else if (!HasChain) + } else if (!HasChain) { Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurDebugLoc(), VTs, &Ops[0], Ops.size()); - else if (I.getType() != Type::getVoidTy(*DAG.getContext())) + } else if (I.getType() != Type::getVoidTy(*DAG.getContext())) { Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurDebugLoc(), VTs, &Ops[0], Ops.size()); - else + } else { Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurDebugLoc(), VTs, &Ops[0], Ops.size()); + } + + if (DisableScheduling) + DAG.AssignOrdering(Result.getNode(), SDNodeOrder); if (HasChain) { SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1); @@ -3062,11 +3171,16 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I, else DAG.setRoot(Chain); } + if (I.getType() != Type::getVoidTy(*DAG.getContext())) { if (const VectorType *PTy = dyn_cast(I.getType())) { EVT VT = TLI.getValueType(PTy); Result = DAG.getNode(ISD::BIT_CONVERT, getCurDebugLoc(), VT, Result); + + if (DisableScheduling) + DAG.AssignOrdering(Result.getNode(), SDNodeOrder); } + setValue(&I, Result); } } @@ -3078,12 +3192,20 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I, /// /// where Op is the hexidecimal representation of floating point value. static SDValue -GetSignificand(SelectionDAG &DAG, SDValue Op, DebugLoc dl) { +GetSignificand(SelectionDAG &DAG, SDValue Op, DebugLoc dl, unsigned Order) { SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, DAG.getConstant(0x007fffff, MVT::i32)); SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1, DAG.getConstant(0x3f800000, MVT::i32)); - return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t2); + SDValue Res = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t2); + + if (DisableScheduling) { + DAG.AssignOrdering(t1.getNode(), Order); + DAG.AssignOrdering(t2.getNode(), Order); + DAG.AssignOrdering(Res.getNode(), Order); + } + + return Res; } /// GetExponent - Get the exponent: @@ -3093,14 +3215,23 @@ GetSignificand(SelectionDAG &DAG, SDValue Op, DebugLoc dl) { /// where Op is the hexidecimal representation of floating point value. static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, - DebugLoc dl) { + DebugLoc dl, unsigned Order) { SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, DAG.getConstant(0x7f800000, MVT::i32)); SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0, DAG.getConstant(23, TLI.getPointerTy())); SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1, DAG.getConstant(127, MVT::i32)); - return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); + SDValue Res = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), Order); + DAG.AssignOrdering(t1.getNode(), Order); + DAG.AssignOrdering(t2.getNode(), Order); + DAG.AssignOrdering(Res.getNode(), Order); + } + + return Res; } /// getF32Constant - Get 32-bit floating point constant. @@ -3124,6 +3255,10 @@ SelectionDAGBuilder::implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op) { I.getOperand(1)); setValue(&I, L); DAG.setRoot(L.getValue(1)); + + if (DisableScheduling) + DAG.AssignOrdering(L.getNode(), SDNodeOrder); + return 0; } @@ -3137,6 +3272,10 @@ SelectionDAGBuilder::implVisitAluOverflow(CallInst &I, ISD::NodeType Op) { SDValue Result = DAG.getNode(Op, getCurDebugLoc(), VTs, Op1, Op2); setValue(&I, Result); + + if (DisableScheduling) + DAG.AssignOrdering(Result.getNode(), SDNodeOrder); + return 0; } @@ -3164,10 +3303,20 @@ SelectionDAGBuilder::visitExp(CallInst &I) { SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(X.getNode(), SDNodeOrder); + } + // IntegerPartOfX <<= 23; IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, DAG.getConstant(23, TLI.getPointerTy())); + if (DisableScheduling) + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -3190,6 +3339,16 @@ SelectionDAGBuilder::visitExp(CallInst &I) { TwoToFracPartOfX, IntegerPartOfX); result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t6); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFracPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -3216,6 +3375,18 @@ SelectionDAGBuilder::visitExp(CallInst &I) { TwoToFracPartOfX, IntegerPartOfX); result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t8); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFracPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -3255,12 +3426,32 @@ SelectionDAGBuilder::visitExp(CallInst &I) { TwoToFracPartOfX, IntegerPartOfX); result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t14); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(t9.getNode(), SDNodeOrder); + DAG.AssignOrdering(t10.getNode(), SDNodeOrder); + DAG.AssignOrdering(t11.getNode(), SDNodeOrder); + DAG.AssignOrdering(t12.getNode(), SDNodeOrder); + DAG.AssignOrdering(t13.getNode(), SDNodeOrder); + DAG.AssignOrdering(t14.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFracPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. result = DAG.getNode(ISD::FEXP, dl, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1))); + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); @@ -3278,14 +3469,20 @@ SelectionDAGBuilder::visitLog(CallInst &I) { SDValue Op = getValue(I.getOperand(1)); SDValue Op1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Op); + if (DisableScheduling) + DAG.AssignOrdering(Op1.getNode(), SDNodeOrder); + // Scale the exponent by log(2) [0.69314718f]. - SDValue Exp = GetExponent(DAG, Op1, TLI, dl); + SDValue Exp = GetExponent(DAG, Op1, TLI, dl, SDNodeOrder); SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, getF32Constant(DAG, 0x3f317218)); + if (DisableScheduling) + DAG.AssignOrdering(LogOfExponent.getNode(), SDNodeOrder); + // Get the significand and build it into a floating-point number with // exponent of 1. - SDValue X = GetSignificand(DAG, Op1, dl); + SDValue X = GetSignificand(DAG, Op1, dl, SDNodeOrder); if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: @@ -3305,6 +3502,14 @@ SelectionDAGBuilder::visitLog(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(LogOfMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -3331,6 +3536,18 @@ SelectionDAGBuilder::visitLog(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(LogOfMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -3365,12 +3582,31 @@ SelectionDAGBuilder::visitLog(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(t9.getNode(), SDNodeOrder); + DAG.AssignOrdering(t10.getNode(), SDNodeOrder); + DAG.AssignOrdering(LogOfMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. result = DAG.getNode(ISD::FLOG, dl, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1))); + + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); @@ -3388,12 +3624,18 @@ SelectionDAGBuilder::visitLog2(CallInst &I) { SDValue Op = getValue(I.getOperand(1)); SDValue Op1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Op); + if (DisableScheduling) + DAG.AssignOrdering(Op1.getNode(), SDNodeOrder); + // Get the exponent. - SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl); + SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl, SDNodeOrder); + + if (DisableScheduling) + DAG.AssignOrdering(LogOfExponent.getNode(), SDNodeOrder); // Get the significand and build it into a floating-point number with // exponent of 1. - SDValue X = GetSignificand(DAG, Op1, dl); + SDValue X = GetSignificand(DAG, Op1, dl, SDNodeOrder); // Different possible minimax approximations of significand in // floating-point for various degrees of accuracy over [1,2]. @@ -3413,6 +3655,14 @@ SelectionDAGBuilder::visitLog2(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log2ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -3439,6 +3689,18 @@ SelectionDAGBuilder::visitLog2(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log2ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -3474,12 +3736,31 @@ SelectionDAGBuilder::visitLog2(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(t9.getNode(), SDNodeOrder); + DAG.AssignOrdering(t10.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log2ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. result = DAG.getNode(ISD::FLOG2, dl, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1))); + + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); @@ -3497,14 +3778,20 @@ SelectionDAGBuilder::visitLog10(CallInst &I) { SDValue Op = getValue(I.getOperand(1)); SDValue Op1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Op); + if (DisableScheduling) + DAG.AssignOrdering(Op1.getNode(), SDNodeOrder); + // Scale the exponent by log10(2) [0.30102999f]. - SDValue Exp = GetExponent(DAG, Op1, TLI, dl); + SDValue Exp = GetExponent(DAG, Op1, TLI, dl, SDNodeOrder); SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, getF32Constant(DAG, 0x3e9a209a)); + if (DisableScheduling) + DAG.AssignOrdering(LogOfExponent.getNode(), SDNodeOrder); + // Get the significand and build it into a floating-point number with // exponent of 1. - SDValue X = GetSignificand(DAG, Op1, dl); + SDValue X = GetSignificand(DAG, Op1, dl, SDNodeOrder); if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: @@ -3524,6 +3811,14 @@ SelectionDAGBuilder::visitLog10(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log10ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -3546,6 +3841,16 @@ SelectionDAGBuilder::visitLog10(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log10ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -3576,12 +3881,29 @@ SelectionDAGBuilder::visitLog10(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log10ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. result = DAG.getNode(ISD::FLOG10, dl, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1))); + + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); @@ -3600,6 +3922,9 @@ SelectionDAGBuilder::visitExp2(CallInst &I) { SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op); + if (DisableScheduling) + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + // FractionalPartOfX = x - (float)IntegerPartOfX; SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1); @@ -3608,6 +3933,12 @@ SelectionDAGBuilder::visitExp2(CallInst &I) { IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, DAG.getConstant(23, TLI.getPointerTy())); + if (DisableScheduling) { + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(X.getNode(), SDNodeOrder); + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + } + if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -3629,6 +3960,16 @@ SelectionDAGBuilder::visitExp2(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -3654,6 +3995,18 @@ SelectionDAGBuilder::visitExp2(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -3690,12 +4043,33 @@ SelectionDAGBuilder::visitExp2(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(t9.getNode(), SDNodeOrder); + DAG.AssignOrdering(t10.getNode(), SDNodeOrder); + DAG.AssignOrdering(t11.getNode(), SDNodeOrder); + DAG.AssignOrdering(t12.getNode(), SDNodeOrder); + DAG.AssignOrdering(t13.getNode(), SDNodeOrder); + DAG.AssignOrdering(t14.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. result = DAG.getNode(ISD::FEXP2, dl, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1))); + + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); @@ -3737,10 +4111,20 @@ SelectionDAGBuilder::visitPow(CallInst &I) { SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(X.getNode(), SDNodeOrder); + } + // IntegerPartOfX <<= 23; IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, DAG.getConstant(23, TLI.getPointerTy())); + if (DisableScheduling) + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -3762,6 +4146,16 @@ SelectionDAGBuilder::visitPow(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -3787,6 +4181,18 @@ SelectionDAGBuilder::visitPow(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -3823,6 +4229,24 @@ SelectionDAGBuilder::visitPow(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(t9.getNode(), SDNodeOrder); + DAG.AssignOrdering(t10.getNode(), SDNodeOrder); + DAG.AssignOrdering(t11.getNode(), SDNodeOrder); + DAG.AssignOrdering(t12.getNode(), SDNodeOrder); + DAG.AssignOrdering(t13.getNode(), SDNodeOrder); + DAG.AssignOrdering(t14.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. @@ -3830,17 +4254,75 @@ SelectionDAGBuilder::visitPow(CallInst &I) { getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1)), getValue(I.getOperand(2))); + + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); } + +/// ExpandPowI - Expand a llvm.powi intrinsic. +static SDValue ExpandPowI(DebugLoc DL, SDValue LHS, SDValue RHS, + SelectionDAG &DAG) { + // If RHS is a constant, we can expand this out to a multiplication tree, + // otherwise we end up lowering to a call to __powidf2 (for example). When + // optimizing for size, we only want to do this if the expansion would produce + // a small number of multiplies, otherwise we do the full expansion. + if (ConstantSDNode *RHSC = dyn_cast(RHS)) { + // Get the exponent as a positive value. + unsigned Val = RHSC->getSExtValue(); + if ((int)Val < 0) Val = -Val; + + // powi(x, 0) -> 1.0 + if (Val == 0) + return DAG.getConstantFP(1.0, LHS.getValueType()); + + Function *F = DAG.getMachineFunction().getFunction(); + if (!F->hasFnAttr(Attribute::OptimizeForSize) || + // If optimizing for size, don't insert too many multiplies. This + // inserts up to 5 multiplies. + CountPopulation_32(Val)+Log2_32(Val) < 7) { + // We use the simple binary decomposition method to generate the multiply + // sequence. There are more optimal ways to do this (for example, + // powi(x,15) generates one more multiply than it should), but this has + // the benefit of being both really simple and much better than a libcall. + SDValue Res; // Logically starts equal to 1.0 + SDValue CurSquare = LHS; + while (Val) { + if (Val & 1) + if (Res.getNode()) + Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare); + else + Res = CurSquare; // 1.0*CurSquare. + + CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(), + CurSquare, CurSquare); + Val >>= 1; + } + + // If the original was negative, invert the result, producing 1/(x*x*x). + if (RHSC->getSExtValue() < 0) + Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(), + DAG.getConstantFP(1.0, LHS.getValueType()), Res); + return Res; + } + } + + // Otherwise, expand to a libcall. + return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); +} + + /// visitIntrinsicCall - Lower the call to the specified intrinsic function. If /// we want to emit this as a call to a named external function, return the name /// otherwise lower it and return null. const char * SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DebugLoc dl = getCurDebugLoc(); + SDValue Res; + switch (Intrinsic) { default: // By default, turn this into a target intrinsic node. @@ -3850,26 +4332,33 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::vaend: visitVAEnd(I); return 0; case Intrinsic::vacopy: visitVACopy(I); return 0; case Intrinsic::returnaddress: - setValue(&I, DAG.getNode(ISD::RETURNADDR, dl, TLI.getPointerTy(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::RETURNADDR, dl, TLI.getPointerTy(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::frameaddress: - setValue(&I, DAG.getNode(ISD::FRAMEADDR, dl, TLI.getPointerTy(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::FRAMEADDR, dl, TLI.getPointerTy(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::setjmp: return "_setjmp"+!TLI.usesUnderscoreSetJmp(); - break; case Intrinsic::longjmp: return "_longjmp"+!TLI.usesUnderscoreLongJmp(); - break; case Intrinsic::memcpy: { SDValue Op1 = getValue(I.getOperand(1)); SDValue Op2 = getValue(I.getOperand(2)); SDValue Op3 = getValue(I.getOperand(3)); unsigned Align = cast(I.getOperand(4))->getZExtValue(); - DAG.setRoot(DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false, - I.getOperand(1), 0, I.getOperand(2), 0)); + Res = DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false, + I.getOperand(1), 0, I.getOperand(2), 0); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::memset: { @@ -3877,8 +4366,11 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { SDValue Op2 = getValue(I.getOperand(2)); SDValue Op3 = getValue(I.getOperand(3)); unsigned Align = cast(I.getOperand(4))->getZExtValue(); - DAG.setRoot(DAG.getMemset(getRoot(), dl, Op1, Op2, Op3, Align, - I.getOperand(1), 0)); + Res = DAG.getMemset(getRoot(), dl, Op1, Op2, Op3, Align, + I.getOperand(1), 0); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::memmove: { @@ -3894,13 +4386,19 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { Size = C->getZExtValue(); if (AA->alias(I.getOperand(1), Size, I.getOperand(2), Size) == AliasAnalysis::NoAlias) { - DAG.setRoot(DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false, - I.getOperand(1), 0, I.getOperand(2), 0)); + Res = DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false, + I.getOperand(1), 0, I.getOperand(2), 0); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } - DAG.setRoot(DAG.getMemmove(getRoot(), dl, Op1, Op2, Op3, Align, - I.getOperand(1), 0, I.getOperand(2), 0)); + Res = DAG.getMemmove(getRoot(), dl, Op1, Op2, Op3, Align, + I.getOperand(1), 0, I.getOperand(2), 0); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::dbg_stoppoint: @@ -3917,7 +4415,7 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { if (!DW) return 0; DbgDeclareInst &DI = cast(I); - if (!isValidDebugInfoIntrinsic(DI, CodeGenOpt::None)) + if (!DIDescriptor::ValidDebugInfo(DI.getVariable(), CodeGenOpt::None)) return 0; MDNode *Variable = DI.getVariable(); @@ -3934,14 +4432,9 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return 0; // VLAs. int FI = SI->second; - MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - if (MMI) { - MetadataContext &TheMetadata = - DI.getParent()->getContext().getMetadata(); - unsigned MDDbgKind = TheMetadata.getMDKind("dbg"); - MDNode *Dbg = TheMetadata.getMD(MDDbgKind, &DI); - MMI->setVariableDbgInfo(Variable, FI, Dbg); - } + if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo()) + if (MDNode *Dbg = DI.getMetadata("dbg")) + MMI->setVariableDbgInfo(Variable, FI, Dbg); return 0; } case Intrinsic::eh_exception: { @@ -3953,6 +4446,8 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { SDValue Op = DAG.getNode(ISD::EXCEPTIONADDR, dl, VTs, Ops, 1); setValue(&I, Op); DAG.setRoot(Op.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(Op.getNode(), SDNodeOrder); return 0; } @@ -3979,7 +4474,12 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DAG.setRoot(Op.getValue(1)); - setValue(&I, DAG.getSExtOrTrunc(Op, dl, MVT::i32)); + Res = DAG.getSExtOrTrunc(Op, dl, MVT::i32); + setValue(&I, Res); + if (DisableScheduling) { + DAG.AssignOrdering(Op.getNode(), SDNodeOrder); + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + } return 0; } @@ -3989,14 +4489,16 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { if (MMI) { // Find the type id for the given typeinfo. GlobalVariable *GV = ExtractTypeInfo(I.getOperand(1)); - unsigned TypeID = MMI->getTypeIDFor(GV); - setValue(&I, DAG.getConstant(TypeID, MVT::i32)); + Res = DAG.getConstant(TypeID, MVT::i32); } else { // Return something different to eh_selector. - setValue(&I, DAG.getConstant(1, MVT::i32)); + Res = DAG.getConstant(1, MVT::i32); } + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } @@ -4004,11 +4506,14 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::eh_return_i64: if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo()) { MMI->setCallsEHReturn(true); - DAG.setRoot(DAG.getNode(ISD::EH_RETURN, dl, - MVT::Other, - getControlRoot(), - getValue(I.getOperand(1)), - getValue(I.getOperand(2)))); + Res = DAG.getNode(ISD::EH_RETURN, dl, + MVT::Other, + getControlRoot(), + getValue(I.getOperand(1)), + getValue(I.getOperand(2))); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } else { setValue(&I, DAG.getConstant(0, TLI.getPointerTy())); } @@ -4018,26 +4523,28 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo()) { MMI->setCallsUnwindInit(true); } - return 0; - case Intrinsic::eh_dwarf_cfa: { EVT VT = getValue(I.getOperand(1)).getValueType(); SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), dl, TLI.getPointerTy()); - SDValue Offset = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, TLI.getPointerTy()), CfaArg); - setValue(&I, DAG.getNode(ISD::ADD, dl, + SDValue FA = DAG.getNode(ISD::FRAMEADDR, dl, TLI.getPointerTy(), - DAG.getNode(ISD::FRAMEADDR, dl, - TLI.getPointerTy(), - DAG.getConstant(0, - TLI.getPointerTy())), - Offset)); + DAG.getConstant(0, TLI.getPointerTy())); + Res = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), + FA, Offset); + setValue(&I, Res); + if (DisableScheduling) { + DAG.AssignOrdering(CfaArg.getNode(), SDNodeOrder); + DAG.AssignOrdering(Offset.getNode(), SDNodeOrder); + DAG.AssignOrdering(FA.getNode(), SDNodeOrder); + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + } return 0; } case Intrinsic::convertff: @@ -4062,36 +4569,48 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::convertuu: Code = ISD::CVT_UU; break; } EVT DestVT = TLI.getValueType(I.getType()); - Value* Op1 = I.getOperand(1); - setValue(&I, DAG.getConvertRndSat(DestVT, getCurDebugLoc(), getValue(Op1), - DAG.getValueType(DestVT), - DAG.getValueType(getValue(Op1).getValueType()), - getValue(I.getOperand(2)), - getValue(I.getOperand(3)), - Code)); + Value *Op1 = I.getOperand(1); + Res = DAG.getConvertRndSat(DestVT, getCurDebugLoc(), getValue(Op1), + DAG.getValueType(DestVT), + DAG.getValueType(getValue(Op1).getValueType()), + getValue(I.getOperand(2)), + getValue(I.getOperand(3)), + Code); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } - case Intrinsic::sqrt: - setValue(&I, DAG.getNode(ISD::FSQRT, dl, - getValue(I.getOperand(1)).getValueType(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::FSQRT, dl, + getValue(I.getOperand(1)).getValueType(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::powi: - setValue(&I, DAG.getNode(ISD::FPOWI, dl, - getValue(I.getOperand(1)).getValueType(), - getValue(I.getOperand(1)), - getValue(I.getOperand(2)))); + Res = ExpandPowI(dl, getValue(I.getOperand(1)), getValue(I.getOperand(2)), + DAG); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::sin: - setValue(&I, DAG.getNode(ISD::FSIN, dl, - getValue(I.getOperand(1)).getValueType(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::FSIN, dl, + getValue(I.getOperand(1)).getValueType(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::cos: - setValue(&I, DAG.getNode(ISD::FCOS, dl, - getValue(I.getOperand(1)).getValueType(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::FCOS, dl, + getValue(I.getOperand(1)).getValueType(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::log: visitLog(I); @@ -4113,55 +4632,74 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return 0; case Intrinsic::pcmarker: { SDValue Tmp = getValue(I.getOperand(1)); - DAG.setRoot(DAG.getNode(ISD::PCMARKER, dl, MVT::Other, getRoot(), Tmp)); + Res = DAG.getNode(ISD::PCMARKER, dl, MVT::Other, getRoot(), Tmp); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::readcyclecounter: { SDValue Op = getRoot(); - SDValue Tmp = DAG.getNode(ISD::READCYCLECOUNTER, dl, - DAG.getVTList(MVT::i64, MVT::Other), - &Op, 1); - setValue(&I, Tmp); - DAG.setRoot(Tmp.getValue(1)); + Res = DAG.getNode(ISD::READCYCLECOUNTER, dl, + DAG.getVTList(MVT::i64, MVT::Other), + &Op, 1); + setValue(&I, Res); + DAG.setRoot(Res.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::bswap: - setValue(&I, DAG.getNode(ISD::BSWAP, dl, - getValue(I.getOperand(1)).getValueType(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::BSWAP, dl, + getValue(I.getOperand(1)).getValueType(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::cttz: { SDValue Arg = getValue(I.getOperand(1)); EVT Ty = Arg.getValueType(); - SDValue result = DAG.getNode(ISD::CTTZ, dl, Ty, Arg); - setValue(&I, result); + Res = DAG.getNode(ISD::CTTZ, dl, Ty, Arg); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::ctlz: { SDValue Arg = getValue(I.getOperand(1)); EVT Ty = Arg.getValueType(); - SDValue result = DAG.getNode(ISD::CTLZ, dl, Ty, Arg); - setValue(&I, result); + Res = DAG.getNode(ISD::CTLZ, dl, Ty, Arg); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::ctpop: { SDValue Arg = getValue(I.getOperand(1)); EVT Ty = Arg.getValueType(); - SDValue result = DAG.getNode(ISD::CTPOP, dl, Ty, Arg); - setValue(&I, result); + Res = DAG.getNode(ISD::CTPOP, dl, Ty, Arg); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::stacksave: { SDValue Op = getRoot(); - SDValue Tmp = DAG.getNode(ISD::STACKSAVE, dl, - DAG.getVTList(TLI.getPointerTy(), MVT::Other), &Op, 1); - setValue(&I, Tmp); - DAG.setRoot(Tmp.getValue(1)); + Res = DAG.getNode(ISD::STACKSAVE, dl, + DAG.getVTList(TLI.getPointerTy(), MVT::Other), &Op, 1); + setValue(&I, Res); + DAG.setRoot(Res.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::stackrestore: { - SDValue Tmp = getValue(I.getOperand(1)); - DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, dl, MVT::Other, getRoot(), Tmp)); + Res = getValue(I.getOperand(1)); + Res = DAG.getNode(ISD::STACKRESTORE, dl, MVT::Other, getRoot(), Res); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::stackprotector: { @@ -4179,11 +4717,13 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { SDValue FIN = DAG.getFrameIndex(FI, PtrTy); // Store the stack protector onto the stack. - SDValue Result = DAG.getStore(getRoot(), getCurDebugLoc(), Src, FIN, - PseudoSourceValue::getFixedStack(FI), - 0, true); - setValue(&I, Result); - DAG.setRoot(Result); + Res = DAG.getStore(getRoot(), getCurDebugLoc(), Src, FIN, + PseudoSourceValue::getFixedStack(FI), + 0, true); + setValue(&I, Res); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::objectsize: { @@ -4195,10 +4735,14 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { SDValue Arg = getValue(I.getOperand(0)); EVT Ty = Arg.getValueType(); - if (CI->getZExtValue() < 2) - setValue(&I, DAG.getConstant(-1ULL, Ty)); + if (CI->getZExtValue() == 0) + Res = DAG.getConstant(-1ULL, Ty); else - setValue(&I, DAG.getConstant(0, Ty)); + Res = DAG.getConstant(0, Ty); + + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::var_annotation: @@ -4216,15 +4760,16 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { Ops[4] = DAG.getSrcValue(I.getOperand(1)); Ops[5] = DAG.getSrcValue(F); - SDValue Tmp = DAG.getNode(ISD::TRAMPOLINE, dl, - DAG.getVTList(TLI.getPointerTy(), MVT::Other), - Ops, 6); + Res = DAG.getNode(ISD::TRAMPOLINE, dl, + DAG.getVTList(TLI.getPointerTy(), MVT::Other), + Ops, 6); - setValue(&I, Tmp); - DAG.setRoot(Tmp.getValue(1)); + setValue(&I, Res); + DAG.setRoot(Res.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } - case Intrinsic::gcroot: if (GFI) { Value *Alloca = I.getOperand(1); @@ -4234,22 +4779,22 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { GFI->addStackRoot(FI->getIndex(), TypeMap); } return 0; - case Intrinsic::gcread: case Intrinsic::gcwrite: llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!"); return 0; - - case Intrinsic::flt_rounds: { - setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, dl, MVT::i32)); + case Intrinsic::flt_rounds: + Res = DAG.getNode(ISD::FLT_ROUNDS_, dl, MVT::i32); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; - } - - case Intrinsic::trap: { - DAG.setRoot(DAG.getNode(ISD::TRAP, dl,MVT::Other, getRoot())); + case Intrinsic::trap: + Res = DAG.getNode(ISD::TRAP, dl,MVT::Other, getRoot()); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; - } - case Intrinsic::uadd_with_overflow: return implVisitAluOverflow(I, ISD::UADDO); case Intrinsic::sadd_with_overflow: @@ -4269,7 +4814,10 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { Ops[1] = getValue(I.getOperand(1)); Ops[2] = getValue(I.getOperand(2)); Ops[3] = getValue(I.getOperand(3)); - DAG.setRoot(DAG.getNode(ISD::PREFETCH, dl, MVT::Other, &Ops[0], 4)); + Res = DAG.getNode(ISD::PREFETCH, dl, MVT::Other, &Ops[0], 4); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } @@ -4279,7 +4827,10 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { for (int x = 1; x < 6; ++x) Ops[x] = getValue(I.getOperand(x)); - DAG.setRoot(DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, &Ops[0], 6)); + Res = DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, &Ops[0], 6); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::atomic_cmp_swap: { @@ -4294,6 +4845,8 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { I.getOperand(1)); setValue(&I, L); DAG.setRoot(L.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(L.getNode(), SDNodeOrder); return 0; } case Intrinsic::atomic_load_add: @@ -4322,7 +4875,10 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::invariant_start: case Intrinsic::lifetime_start: // Discard region information. - setValue(&I, DAG.getUNDEF(TLI.getPointerTy())); + Res = DAG.getUNDEF(TLI.getPointerTy()); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::invariant_end: case Intrinsic::lifetime_end: @@ -4422,8 +4978,7 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee, SmallVector OutsFlags; SmallVector Offsets; getReturnInfo(RetTy, CS.getAttributes().getRetAttributes(), - OutVTs, OutsFlags, TLI, &Offsets); - + OutVTs, OutsFlags, TLI, &Offsets); bool CanLowerReturn = TLI.CanLowerReturn(CS.getCallingConv(), FTy->isVarArg(), OutVTs, OutsFlags, DAG); @@ -4497,14 +5052,16 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee, CS.getCallingConv(), isTailCall, !CS.getInstruction()->use_empty(), - Callee, Args, DAG, getCurDebugLoc()); + Callee, Args, DAG, getCurDebugLoc(), SDNodeOrder); assert((isTailCall || Result.second.getNode()) && "Non-null chain expected with non-tail call!"); assert((Result.second.getNode() || !Result.first.getNode()) && "Null value expected with tail call!"); - if (Result.first.getNode()) + if (Result.first.getNode()) { setValue(CS.getInstruction(), Result.first); - else if (!CanLowerReturn && Result.second.getNode()) { + if (DisableScheduling) + DAG.AssignOrdering(Result.first.getNode(), SDNodeOrder); + } else if (!CanLowerReturn && Result.second.getNode()) { // The instruction result is the result of loading from the // hidden sret parameter. SmallVector PVTs; @@ -4518,27 +5075,40 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee, SmallVector Chains(NumValues); for (unsigned i = 0; i < NumValues; ++i) { + SDValue Add = DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, + DemoteStackSlot, + DAG.getConstant(Offsets[i], PtrVT)); SDValue L = DAG.getLoad(OutVTs[i], getCurDebugLoc(), Result.second, - DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, DemoteStackSlot, - DAG.getConstant(Offsets[i], PtrVT)), - NULL, Offsets[i], false, 1); + Add, NULL, Offsets[i], false, 1); Values[i] = L; Chains[i] = L.getValue(1); } + SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), MVT::Other, &Chains[0], NumValues); PendingLoads.push_back(Chain); - setValue(CS.getInstruction(), DAG.getNode(ISD::MERGE_VALUES, - getCurDebugLoc(), DAG.getVTList(&OutVTs[0], NumValues), - &Values[0], NumValues)); + SDValue MV = DAG.getNode(ISD::MERGE_VALUES, + getCurDebugLoc(), + DAG.getVTList(&OutVTs[0], NumValues), + &Values[0], NumValues); + setValue(CS.getInstruction(), MV); + + if (DisableScheduling) { + DAG.AssignOrdering(Chain.getNode(), SDNodeOrder); + DAG.AssignOrdering(MV.getNode(), SDNodeOrder); + } } - // As a special case, a null chain means that a tail call has - // been emitted and the DAG root is already updated. - if (Result.second.getNode()) + + // As a special case, a null chain means that a tail call has been emitted and + // the DAG root is already updated. + if (Result.second.getNode()) { DAG.setRoot(Result.second); - else + if (DisableScheduling) + DAG.AssignOrdering(Result.second.getNode(), SDNodeOrder); + } else { HasTailCall = true; + } if (LandingPad && MMI) { // Insert a label at the end of the invoke call to mark the try range. This @@ -4552,6 +5122,140 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee, } } +/// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the +/// value is equal or not-equal to zero. +static bool IsOnlyUsedInZeroEqualityComparison(Value *V) { + for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); + UI != E; ++UI) { + if (ICmpInst *IC = dyn_cast(*UI)) + if (IC->isEquality()) + if (Constant *C = dyn_cast(IC->getOperand(1))) + if (C->isNullValue()) + continue; + // Unknown instruction. + return false; + } + return true; +} + +static SDValue getMemCmpLoad(Value *PtrVal, MVT LoadVT, const Type *LoadTy, + SelectionDAGBuilder &Builder) { + + // Check to see if this load can be trivially constant folded, e.g. if the + // input is from a string literal. + if (Constant *LoadInput = dyn_cast(PtrVal)) { + // Cast pointer to the type we really want to load. + LoadInput = ConstantExpr::getBitCast(LoadInput, + PointerType::getUnqual(LoadTy)); + + if (Constant *LoadCst = ConstantFoldLoadFromConstPtr(LoadInput, Builder.TD)) + return Builder.getValue(LoadCst); + } + + // Otherwise, we have to emit the load. If the pointer is to unfoldable but + // still constant memory, the input chain can be the entry node. + SDValue Root; + bool ConstantMemory = false; + + // Do not serialize (non-volatile) loads of constant memory with anything. + if (Builder.AA->pointsToConstantMemory(PtrVal)) { + Root = Builder.DAG.getEntryNode(); + ConstantMemory = true; + } else { + // Do not serialize non-volatile loads against each other. + Root = Builder.DAG.getRoot(); + } + + SDValue Ptr = Builder.getValue(PtrVal); + SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurDebugLoc(), Root, + Ptr, PtrVal /*SrcValue*/, 0/*SVOffset*/, + false /*volatile*/, 1 /* align=1 */); + + if (!ConstantMemory) + Builder.PendingLoads.push_back(LoadVal.getValue(1)); + return LoadVal; +} + + +/// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form. +/// If so, return true and lower it, otherwise return false and it will be +/// lowered like a normal call. +bool SelectionDAGBuilder::visitMemCmpCall(CallInst &I) { + // Verify that the prototype makes sense. int memcmp(void*,void*,size_t) + if (I.getNumOperands() != 4) + return false; + + Value *LHS = I.getOperand(1), *RHS = I.getOperand(2); + if (!isa(LHS->getType()) || !isa(RHS->getType()) || + !isa(I.getOperand(3)->getType()) || + !isa(I.getType())) + return false; + + ConstantInt *Size = dyn_cast(I.getOperand(3)); + + // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0 + // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0 + if (Size && IsOnlyUsedInZeroEqualityComparison(&I)) { + bool ActuallyDoIt = true; + MVT LoadVT; + const Type *LoadTy; + switch (Size->getZExtValue()) { + default: + LoadVT = MVT::Other; + LoadTy = 0; + ActuallyDoIt = false; + break; + case 2: + LoadVT = MVT::i16; + LoadTy = Type::getInt16Ty(Size->getContext()); + break; + case 4: + LoadVT = MVT::i32; + LoadTy = Type::getInt32Ty(Size->getContext()); + break; + case 8: + LoadVT = MVT::i64; + LoadTy = Type::getInt64Ty(Size->getContext()); + break; + /* + case 16: + LoadVT = MVT::v4i32; + LoadTy = Type::getInt32Ty(Size->getContext()); + LoadTy = VectorType::get(LoadTy, 4); + break; + */ + } + + // This turns into unaligned loads. We only do this if the target natively + // supports the MVT we'll be loading or if it is small enough (<= 4) that + // we'll only produce a small number of byte loads. + + // Require that we can find a legal MVT, and only do this if the target + // supports unaligned loads of that type. Expanding into byte loads would + // bloat the code. + if (ActuallyDoIt && Size->getZExtValue() > 4) { + // TODO: Handle 5 byte compare as 4-byte + 1 byte. + // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads. + if (!TLI.isTypeLegal(LoadVT) ||!TLI.allowsUnalignedMemoryAccesses(LoadVT)) + ActuallyDoIt = false; + } + + if (ActuallyDoIt) { + SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this); + SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this); + + SDValue Res = DAG.getSetCC(getCurDebugLoc(), MVT::i1, LHSVal, RHSVal, + ISD::SETNE); + EVT CallVT = TLI.getValueType(I.getType(), true); + setValue(&I, DAG.getZExtOrTrunc(Res, getCurDebugLoc(), CallVT)); + return true; + } + } + + + return false; +} + void SelectionDAGBuilder::visitCall(CallInst &I) { const char *RenameFn = 0; @@ -4626,6 +5330,9 @@ void SelectionDAGBuilder::visitCall(CallInst &I) { Tmp.getValueType(), Tmp)); return; } + } else if (Name == "memcmp") { + if (visitMemCmpCall(I)) + return; } } } else if (isa(I.getOperand(0))) { @@ -4639,21 +5346,19 @@ void SelectionDAGBuilder::visitCall(CallInst &I) { else Callee = DAG.getExternalSymbol(RenameFn, TLI.getPointerTy()); - // Check if we can potentially perform a tail call. More detailed - // checking is be done within LowerCallTo, after more information - // about the call is known. + // Check if we can potentially perform a tail call. More detailed checking is + // be done within LowerCallTo, after more information about the call is known. bool isTailCall = PerformTailCallOpt && I.isTailCall(); LowerCallTo(&I, Callee, isTailCall); } - /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from /// this value and returns the result as a ValueVT value. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, - SDValue &Chain, + unsigned Order, SDValue &Chain, SDValue *Flag) const { // Assemble the legal parts into the final values. SmallVector Values(ValueVTs.size()); @@ -4667,14 +5372,18 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, Parts.resize(NumRegs); for (unsigned i = 0; i != NumRegs; ++i) { SDValue P; - if (Flag == 0) + if (Flag == 0) { P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); - else { + } else { P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag); *Flag = P.getValue(2); } + Chain = P.getValue(1); + if (DisableScheduling) + DAG.AssignOrdering(P.getNode(), Order); + // If the source register was virtual and if we know something about it, // add an assert node. if (TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) && @@ -4713,6 +5422,8 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, P = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl, RegisterVT, P, DAG.getValueType(FromVT)); + if (DisableScheduling) + DAG.AssignOrdering(P.getNode(), Order); } } } @@ -4720,15 +5431,20 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, Parts[i] = P; } - Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), + Values[Value] = getCopyFromParts(DAG, dl, Order, Parts.begin(), NumRegs, RegisterVT, ValueVT); + if (DisableScheduling) + DAG.AssignOrdering(Values[Value].getNode(), Order); Part += NumRegs; Parts.clear(); } - return DAG.getNode(ISD::MERGE_VALUES, dl, - DAG.getVTList(&ValueVTs[0], ValueVTs.size()), - &Values[0], ValueVTs.size()); + SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl, + DAG.getVTList(&ValueVTs[0], ValueVTs.size()), + &Values[0], ValueVTs.size()); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); + return Res; } /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the @@ -4736,7 +5452,8 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, - SDValue &Chain, SDValue *Flag) const { + unsigned Order, SDValue &Chain, + SDValue *Flag) const { // Get the list of the values's legal parts. unsigned NumRegs = Regs.size(); SmallVector Parts(NumRegs); @@ -4745,7 +5462,8 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, unsigned NumParts = TLI->getNumRegisters(*DAG.getContext(), ValueVT); EVT RegisterVT = RegVTs[Value]; - getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), + getCopyToParts(DAG, dl, Order, + Val.getValue(Val.getResNo() + Value), &Parts[Part], NumParts, RegisterVT); Part += NumParts; } @@ -4754,13 +5472,17 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, SmallVector Chains(NumRegs); for (unsigned i = 0; i != NumRegs; ++i) { SDValue Part; - if (Flag == 0) + if (Flag == 0) { Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]); - else { + } else { Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag); *Flag = Part.getValue(1); } + Chains[i] = Part.getValue(0); + + if (DisableScheduling) + DAG.AssignOrdering(Part.getNode(), Order); } if (NumRegs == 1 || Flag) @@ -4777,6 +5499,9 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, Chain = Chains[NumRegs-1]; else Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Chains[0], NumRegs); + + if (DisableScheduling) + DAG.AssignOrdering(Chain.getNode(), Order); } /// AddInlineAsmOperands - Add this value to the specified inlineasm node @@ -4784,20 +5509,28 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, /// values added into it. void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,unsigned MatchingIdx, - SelectionDAG &DAG, + SelectionDAG &DAG, unsigned Order, std::vector &Ops) const { - EVT IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); assert(Regs.size() < (1 << 13) && "Too many inline asm outputs!"); unsigned Flag = Code | (Regs.size() << 3); if (HasMatching) Flag |= 0x80000000 | (MatchingIdx << 16); - Ops.push_back(DAG.getTargetConstant(Flag, IntPtrTy)); + SDValue Res = DAG.getTargetConstant(Flag, MVT::i32); + Ops.push_back(Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); + for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { unsigned NumRegs = TLI->getNumRegisters(*DAG.getContext(), ValueVTs[Value]); EVT RegisterVT = RegVTs[Value]; for (unsigned i = 0; i != NumRegs; ++i) { assert(Reg < Regs.size() && "Mismatch in # registers expected"); - Ops.push_back(DAG.getRegister(Regs[Reg++], RegisterVT)); + SDValue Res = DAG.getRegister(Regs[Reg++], RegisterVT); + Ops.push_back(Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); } } } @@ -4901,8 +5634,12 @@ public: // If this is an indirect operand, the operand is a pointer to the // accessed type. - if (isIndirect) - OpTy = cast(OpTy)->getElementType(); + if (isIndirect) { + const llvm::PointerType *PtrTy = dyn_cast(OpTy); + if (!PtrTy) + llvm_report_error("Indirect operand for inline asm not a pointer!"); + OpTy = PtrTy->getElementType(); + } // If OpTy is not a single value, it may be a struct/union that we // can tile with integers. @@ -5012,6 +5749,9 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, RegVT, OpInfo.CallOperand); OpInfo.ConstraintVT = RegVT; } + + if (DisableScheduling) + DAG.AssignOrdering(OpInfo.CallOperand.getNode(), SDNodeOrder); } NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT); @@ -5048,6 +5788,7 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, Regs.push_back(*I); } } + OpInfo.AssignedRegs = RegsForValue(TLI, Regs, RegVT, ValueVT); const TargetRegisterInfo *TRI = DAG.getTarget().getRegisterInfo(); OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs, *TRI); @@ -5282,6 +6023,7 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // There is no longer a Value* corresponding to this operand. OpInfo.CallOperandVal = 0; + // It is now an indirect operand. OpInfo.isIndirect = true; } @@ -5291,8 +6033,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { if (OpInfo.ConstraintType == TargetLowering::C_Register) GetRegistersForValue(OpInfo, OutputRegs, InputRegs); } - ConstraintInfos.clear(); + ConstraintInfos.clear(); // Second pass - Loop over all of the operands, assigning virtual or physregs // to register class operands. @@ -5366,7 +6108,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { 2 /* REGDEF */ , false, 0, - DAG, AsmNodeOperands); + DAG, SDNodeOrder, + AsmNodeOperands); break; } case InlineAsm::isInput: { @@ -5413,10 +6156,10 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // Use the produced MatchedRegs object to MatchedRegs.getCopyToRegs(InOperandVal, DAG, getCurDebugLoc(), - Chain, &Flag); + SDNodeOrder, Chain, &Flag); MatchedRegs.AddInlineAsmOperands(1 /*REGUSE*/, true, OpInfo.getMatchedOperand(), - DAG, AsmNodeOperands); + DAG, SDNodeOrder, AsmNodeOperands); break; } else { assert(((OpFlag & 7) == 4) && "Unknown matching constraint!"); @@ -5476,10 +6219,11 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { } OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, getCurDebugLoc(), - Chain, &Flag); + SDNodeOrder, Chain, &Flag); OpInfo.AssignedRegs.AddInlineAsmOperands(1/*REGUSE*/, false, 0, - DAG, AsmNodeOperands); + DAG, SDNodeOrder, + AsmNodeOperands); break; } case InlineAsm::isClobber: { @@ -5487,7 +6231,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // allocator is aware that the physreg got clobbered. if (!OpInfo.AssignedRegs.Regs.empty()) OpInfo.AssignedRegs.AddInlineAsmOperands(6 /* EARLYCLOBBER REGDEF */, - false, 0, DAG,AsmNodeOperands); + false, 0, DAG, SDNodeOrder, + AsmNodeOperands); break; } } @@ -5506,7 +6251,7 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // and set it as the value of the call. if (!RetValRegs.Regs.empty()) { SDValue Val = RetValRegs.getCopyFromRegs(DAG, getCurDebugLoc(), - Chain, &Flag); + SDNodeOrder, Chain, &Flag); // FIXME: Why don't we do this for inline asms with MRVs? if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) { @@ -5546,21 +6291,25 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { RegsForValue &OutRegs = IndirectStoresToEmit[i].first; Value *Ptr = IndirectStoresToEmit[i].second; SDValue OutVal = OutRegs.getCopyFromRegs(DAG, getCurDebugLoc(), - Chain, &Flag); + SDNodeOrder, Chain, &Flag); StoresToEmit.push_back(std::make_pair(OutVal, Ptr)); } // Emit the non-flagged stores from the physregs. SmallVector OutChains; - for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) - OutChains.push_back(DAG.getStore(Chain, getCurDebugLoc(), - StoresToEmit[i].first, - getValue(StoresToEmit[i].second), - StoresToEmit[i].second, 0)); + for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) { + SDValue Val = DAG.getStore(Chain, getCurDebugLoc(), + StoresToEmit[i].first, + getValue(StoresToEmit[i].second), + StoresToEmit[i].second, 0); + OutChains.push_back(Val); + } + if (!OutChains.empty()) Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), MVT::Other, &OutChains[0], OutChains.size()); + DAG.setRoot(Chain); } @@ -5606,8 +6355,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, CallingConv::ID CallConv, bool isTailCall, bool isReturnValueUsed, SDValue Callee, - ArgListTy &Args, SelectionDAG &DAG, DebugLoc dl) { - + ArgListTy &Args, SelectionDAG &DAG, DebugLoc dl, + unsigned Order) { assert((!isTailCall || PerformTailCallOpt) && "isTailCall set when tail-call optimizations are disabled!"); @@ -5661,7 +6410,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, else if (Args[i].isZExt) ExtendKind = ISD::ZERO_EXTEND; - getCopyToParts(DAG, dl, Op, &Parts[0], NumParts, PartVT, ExtendKind); + getCopyToParts(DAG, dl, Order, Op, &Parts[0], NumParts, + PartVT, ExtendKind); for (unsigned j = 0; j != NumParts; ++j) { // if it isn't first piece, alignment must be 1 @@ -5722,6 +6472,9 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, "LowerCall emitted a value with the wrong type!"); }); + if (DisableScheduling) + DAG.AssignOrdering(Chain.getNode(), Order); + // For a tail call, the return value is merely live-out and there aren't // any nodes in the DAG representing it. Return a special value to // indicate that a tail call has been emitted and no more Instructions @@ -5746,9 +6499,11 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, unsigned NumRegs = getNumRegisters(RetTy->getContext(), VT); SDValue ReturnValue = - getCopyFromParts(DAG, dl, &InVals[CurReg], NumRegs, RegisterVT, VT, - AssertOp); + getCopyFromParts(DAG, dl, Order, &InVals[CurReg], NumRegs, + RegisterVT, VT, AssertOp); ReturnValues.push_back(ReturnValue); + if (DisableScheduling) + DAG.AssignOrdering(ReturnValue.getNode(), Order); CurReg += NumRegs; } @@ -5761,7 +6516,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(&RetTys[0], RetTys.size()), &ReturnValues[0], ReturnValues.size()); - + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); return std::make_pair(Res, Chain); } @@ -5778,7 +6534,6 @@ SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) { return SDValue(); } - void SelectionDAGBuilder::CopyValueToVirtualRegister(Value *V, unsigned Reg) { SDValue Op = getValue(V); assert((Op.getOpcode() != ISD::CopyFromReg || @@ -5788,7 +6543,7 @@ void SelectionDAGBuilder::CopyValueToVirtualRegister(Value *V, unsigned Reg) { RegsForValue RFV(V->getContext(), TLI, Reg, V->getType()); SDValue Chain = DAG.getEntryNode(); - RFV.getCopyToRegs(Op, DAG, getCurDebugLoc(), Chain, 0); + RFV.getCopyToRegs(Op, DAG, getCurDebugLoc(), SDNodeOrder, Chain, 0); PendingExports.push_back(Chain); } @@ -5811,7 +6566,7 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); FLI.CanLowerReturn = TLI.CanLowerReturn(F.getCallingConv(), F.isVarArg(), - OutVTs, OutsFlags, DAG); + OutVTs, OutsFlags, DAG); if (!FLI.CanLowerReturn) { // Put in an sret pointer parameter before all the other parameters. SmallVector ValueVTs; @@ -5891,12 +6646,14 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { "LowerFormalArguments didn't return a valid chain!"); assert(InVals.size() == Ins.size() && "LowerFormalArguments didn't emit the correct number of values!"); - DEBUG(for (unsigned i = 0, e = Ins.size(); i != e; ++i) { - assert(InVals[i].getNode() && - "LowerFormalArguments emitted a null value!"); - assert(Ins[i].VT == InVals[i].getValueType() && - "LowerFormalArguments emitted a value with the wrong type!"); - }); + DEBUG({ + for (unsigned i = 0, e = Ins.size(); i != e; ++i) { + assert(InVals[i].getNode() && + "LowerFormalArguments emitted a null value!"); + assert(Ins[i].VT == InVals[i].getValueType() && + "LowerFormalArguments emitted a value with the wrong type!"); + } + }); // Update the DAG with the new chain value resulting from argument lowering. DAG.setRoot(NewRoot); @@ -5912,8 +6669,8 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { EVT VT = ValueVTs[0]; EVT RegVT = TLI.getRegisterType(*CurDAG->getContext(), VT); ISD::NodeType AssertOp = ISD::DELETED_NODE; - SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, - VT, AssertOp); + SDValue ArgValue = getCopyFromParts(DAG, dl, 0, &InVals[0], 1, + RegVT, VT, AssertOp); MachineFunction& MF = SDB->DAG.getMachineFunction(); MachineRegisterInfo& RegInfo = MF.getRegInfo(); @@ -5921,11 +6678,12 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { FLI.DemoteRegister = SRetReg; NewRoot = SDB->DAG.getCopyToReg(NewRoot, SDB->getCurDebugLoc(), SRetReg, ArgValue); DAG.setRoot(NewRoot); - + // i indexes lowered arguments. Bump it past the hidden sret argument. // Idx indexes LLVM arguments. Don't touch it. ++i; } + for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++Idx) { SmallVector ArgValues; @@ -5944,19 +6702,25 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { else if (F.paramHasAttr(Idx, Attribute::ZExt)) AssertOp = ISD::AssertZext; - ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts, - PartVT, VT, AssertOp)); + ArgValues.push_back(getCopyFromParts(DAG, dl, 0, &InVals[i], + NumParts, PartVT, VT, + AssertOp)); } + i += NumParts; } + if (!I->use_empty()) { - SDB->setValue(I, DAG.getMergeValues(&ArgValues[0], NumValues, - SDB->getCurDebugLoc())); + SDValue Res = DAG.getMergeValues(&ArgValues[0], NumValues, + SDB->getCurDebugLoc()); + SDB->setValue(I, Res); + // If this argument is live outside of the entry block, insert a copy from // whereever we got it to the vreg that other BB's will reference it as. SDB->CopyToExportRegsIfNeeded(I); } } + assert(i == InVals.size() && "Argument register count mismatch!"); // Finally, if the target has anything special to do, allow it to do so.