X-Git-Url: http://plrg.eecs.uci.edu/git/?p=oota-llvm.git;a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FLegalizeTypesGeneric.cpp;h=330c31ce0eecfc869fe01f886c0cd437ba74aed8;hp=59b8ab820165fb3067dea8bac74e9f56d7c03f65;hb=716c5d8a308a2257298f1f227edf7f7ae102cf4f;hpb=475871a144eb604ddaf37503397ba0941442e5fb diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp index 59b8ab82016..330c31ce0ee 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp @@ -11,15 +11,20 @@ // The routines here perform legalization when the details of the type (such as // whether it is an integer or a float) do not matter. // Expansion is the act of changing a computation in an illegal type to be a -// computation in two identical registers of a smaller type. +// computation in two identical registers of a smaller type. The Lo/Hi part +// is required to be stored first in memory on little/big-endian machines. // Splitting is the act of changing a computation in an illegal type to be a // computation in two not necessarily identical registers of a smaller type. +// There are no requirements on how the type is represented in memory. // //===----------------------------------------------------------------------===// #include "LegalizeTypes.h" +#include "llvm/IR/DataLayout.h" using namespace llvm; +#define DEBUG_TYPE "legalize-types" + //===----------------------------------------------------------------------===// // Generic Result Expansion. //===----------------------------------------------------------------------===// @@ -27,55 +32,158 @@ using namespace llvm; // These routines assume that the Lo/Hi part is stored first in memory on // little/big-endian machines, followed by the Hi/Lo part. This means that // they cannot be used as is on vectors, for which Lo is always stored first. +void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo, + SDValue &Lo, SDValue &Hi) { + SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); + GetExpandedOp(Op, Lo, Hi); +} -void DAGTypeLegalizer::ExpandRes_BIT_CONVERT(SDNode *N, SDValue &Lo, - SDValue &Hi) { - MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); +void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) { + EVT OutVT = N->getValueType(0); + EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); SDValue InOp = N->getOperand(0); - MVT InVT = InOp.getValueType(); + EVT InVT = InOp.getValueType(); + SDLoc dl(N); // Handle some special cases efficiently. switch (getTypeAction(InVT)) { - default: - assert(false && "Unknown type action!"); - case Legal: - case PromoteInteger: + case TargetLowering::TypeLegal: + case TargetLowering::TypePromoteInteger: break; - case SoftenFloat: + case TargetLowering::TypePromoteFloat: + llvm_unreachable("Bitcast of a promotion-needing float should never need" + "expansion"); + case TargetLowering::TypeSoftenFloat: // Convert the integer operand instead. SplitInteger(GetSoftenedFloat(InOp), Lo, Hi); - Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Lo); - Hi = DAG.getNode(ISD::BIT_CONVERT, NVT, Hi); + Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); + Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); return; - case ExpandInteger: - case ExpandFloat: + case TargetLowering::TypeExpandInteger: + case TargetLowering::TypeExpandFloat: // Convert the expanded pieces of the input. GetExpandedOp(InOp, Lo, Hi); - Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Lo); - Hi = DAG.getNode(ISD::BIT_CONVERT, NVT, Hi); + if (TLI.hasBigEndianPartOrdering(InVT) != + TLI.hasBigEndianPartOrdering(OutVT)) + std::swap(Lo, Hi); + Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); + Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); return; - case SplitVector: - // Convert the split parts of the input if it was split in two. + case TargetLowering::TypeSplitVector: GetSplitVector(InOp, Lo, Hi); - if (Lo.getValueType() == Hi.getValueType()) { - if (TLI.isBigEndian()) - std::swap(Lo, Hi); - Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Lo); - Hi = DAG.getNode(ISD::BIT_CONVERT, NVT, Hi); - return; - } - break; - case ScalarizeVector: + if (TLI.hasBigEndianPartOrdering(OutVT)) + std::swap(Lo, Hi); + Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); + Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); + return; + case TargetLowering::TypeScalarizeVector: // Convert the element instead. SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi); - Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Lo); - Hi = DAG.getNode(ISD::BIT_CONVERT, NVT, Hi); + Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); + Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); + return; + case TargetLowering::TypeWidenVector: { + assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST"); + InOp = GetWidenedVector(InOp); + EVT LoVT, HiVT; + std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT); + std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT); + if (TLI.hasBigEndianPartOrdering(OutVT)) + std::swap(Lo, Hi); + Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); + Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); + return; + } + } + + if (InVT.isVector() && OutVT.isInteger()) { + // Handle cases like i64 = BITCAST v1i64 on x86, where the operand + // is legal but the result is not. + unsigned NumElems = 2; + EVT ElemVT = NOutVT; + EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems); + + // If is not a legal type, try . + while (!isTypeLegal(NVT)) { + unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2; + // If the element size is smaller than byte, bail. + if (NewSizeInBits < 8) + break; + NumElems *= 2; + ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits); + NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems); + } + + if (isTypeLegal(NVT)) { + SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp); + + SmallVector Vals; + for (unsigned i = 0; i < NumElems; ++i) + Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT, + CastInOp, DAG.getConstant(i, dl, + TLI.getVectorIdxTy()))); + + // Build Lo, Hi pair by pairing extracted elements if needed. + unsigned Slot = 0; + for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) { + // Each iteration will BUILD_PAIR two nodes and append the result until + // there are only two nodes left, i.e. Lo and Hi. + SDValue LHS = Vals[Slot]; + SDValue RHS = Vals[Slot + 1]; + + if (TLI.isBigEndian()) + std::swap(LHS, RHS); + + Vals.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, + EVT::getIntegerVT( + *DAG.getContext(), + LHS.getValueType().getSizeInBits() << 1), + LHS, RHS)); + } + Lo = Vals[Slot++]; + Hi = Vals[Slot++]; + + if (TLI.isBigEndian()) + std::swap(Lo, Hi); + return; + } } - // Lower the bit-convert to a store/load from the stack, then expand the load. - SDValue Op = CreateStackStoreLoad(InOp, N->getValueType(0)); - ExpandRes_NormalLoad(Op.Val, Lo, Hi); + // Lower the bit-convert to a store/load from the stack. + assert(NOutVT.isByteSized() && "Expanded type not byte sized!"); + + // Create the stack frame object. Make sure it is aligned for both + // the source and expanded destination types. + unsigned Alignment = + TLI.getDataLayout()->getPrefTypeAlignment(NOutVT. + getTypeForEVT(*DAG.getContext())); + SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment); + int SPFI = cast(StackPtr.getNode())->getIndex(); + MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI); + + // Emit a store to the stack slot. + SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo, + false, false, 0); + + // Load the first half from the stack slot. + Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo, + false, false, false, 0); + + // Increment the pointer to the other half. + unsigned IncrementSize = NOutVT.getSizeInBits() / 8; + StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, + DAG.getConstant(IncrementSize, dl, + StackPtr.getValueType())); + + // Load the second half from the stack slot. + Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr, + PtrInfo.getWithOffset(IncrementSize), false, + false, false, MinAlign(Alignment, IncrementSize)); + + // Handle endianness of the load. + if (TLI.hasBigEndianPartOrdering(OutVT)) + std::swap(Lo, Hi); } void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo, @@ -88,45 +196,50 @@ void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo, void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo, SDValue &Hi) { GetExpandedOp(N->getOperand(0), Lo, Hi); - SDValue Part = cast(N->getOperand(1))->getValue() ? Hi : Lo; + SDValue Part = cast(N->getOperand(1))->getZExtValue() ? + Hi : Lo; assert(Part.getValueType() == N->getValueType(0) && "Type twice as big as expanded type not itself expanded!"); - MVT NVT = TLI.getTypeToTransformTo(N->getValueType(0)); - Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, NVT, Part, - DAG.getConstant(0, TLI.getPointerTy())); - Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, NVT, Part, - DAG.getConstant(1, TLI.getPointerTy())); + GetPairElements(Part, Lo, Hi); } void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue OldVec = N->getOperand(0); unsigned OldElts = OldVec.getValueType().getVectorNumElements(); + EVT OldEltVT = OldVec.getValueType().getVectorElementType(); + SDLoc dl(N); // Convert to a vector of the expanded element type, for example // <3 x i64> -> <6 x i32>. - MVT OldVT = N->getValueType(0); - MVT NewVT = TLI.getTypeToTransformTo(OldVT); + EVT OldVT = N->getValueType(0); + EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); + + if (OldVT != OldEltVT) { + // The result of EXTRACT_VECTOR_ELT may be larger than the element type of + // the input vector. If so, extend the elements of the input vector to the + // same bitwidth as the result before expanding. + assert(OldEltVT.bitsLT(OldVT) && "Result type smaller then element type!"); + EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldElts); + OldVec = DAG.getNode(ISD::ANY_EXTEND, dl, NVecVT, N->getOperand(0)); + } - SDValue NewVec = DAG.getNode(ISD::BIT_CONVERT, - MVT::getVectorVT(NewVT, 2*OldElts), - OldVec); + SDValue NewVec = DAG.getNode(ISD::BITCAST, dl, + EVT::getVectorVT(*DAG.getContext(), + NewVT, 2*OldElts), + OldVec); // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector. SDValue Idx = N->getOperand(1); - // Make sure the type of Idx is big enough to hold the new values. - if (Idx.getValueType().bitsLT(TLI.getPointerTy())) - Idx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), Idx); + Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); + Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); - Idx = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, Idx); - Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, NewVec, Idx); - - Idx = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, - DAG.getConstant(1, Idx.getValueType())); - Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, NewVec, Idx); + Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, + DAG.getConstant(1, dl, Idx.getValueType())); + Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); if (TLI.isBigEndian()) std::swap(Lo, Hi); @@ -135,34 +248,41 @@ void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo, SDValue &Hi) { assert(ISD::isNormalLoad(N) && "This routine only for normal loads!"); + SDLoc dl(N); LoadSDNode *LD = cast(N); - MVT NVT = TLI.getTypeToTransformTo(LD->getValueType(0)); + EVT ValueVT = LD->getValueType(0); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT); SDValue Chain = LD->getChain(); SDValue Ptr = LD->getBasePtr(); - int SVOffset = LD->getSrcValueOffset(); unsigned Alignment = LD->getAlignment(); bool isVolatile = LD->isVolatile(); + bool isNonTemporal = LD->isNonTemporal(); + bool isInvariant = LD->isInvariant(); + AAMDNodes AAInfo = LD->getAAInfo(); assert(NVT.isByteSized() && "Expanded type not byte sized!"); - Lo = DAG.getLoad(NVT, Chain, Ptr, LD->getSrcValue(), SVOffset, - isVolatile, Alignment); + Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(), + isVolatile, isNonTemporal, isInvariant, Alignment, + AAInfo); // Increment the pointer to the other half. unsigned IncrementSize = NVT.getSizeInBits() / 8; - Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, - DAG.getIntPtrConstant(IncrementSize)); - Hi = DAG.getLoad(NVT, Chain, Ptr, LD->getSrcValue(), SVOffset+IncrementSize, - isVolatile, MinAlign(Alignment, IncrementSize)); + Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, + DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); + Hi = DAG.getLoad(NVT, dl, Chain, Ptr, + LD->getPointerInfo().getWithOffset(IncrementSize), + isVolatile, isNonTemporal, isInvariant, + MinAlign(Alignment, IncrementSize), AAInfo); // Build a factor node to remember that this load is independent of the // other one. - Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), + Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), Hi.getValue(1)); // Handle endianness of the load. - if (TLI.isBigEndian()) + if (TLI.hasBigEndianPartOrdering(ValueVT)) std::swap(Lo, Hi); // Modified the chain - switch anything that used the old chain to use @@ -170,31 +290,80 @@ void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo, ReplaceValueWith(SDValue(N, 1), Chain); } +void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { + EVT OVT = N->getValueType(0); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); + SDValue Chain = N->getOperand(0); + SDValue Ptr = N->getOperand(1); + SDLoc dl(N); + const unsigned Align = N->getConstantOperandVal(3); + + Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align); + Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0); + + // Handle endianness of the load. + if (TLI.hasBigEndianPartOrdering(OVT)) + std::swap(Lo, Hi); + + // Modified the chain - switch anything that used the old chain to use + // the new one. + ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); +} + //===--------------------------------------------------------------------===// // Generic Operand Expansion. //===--------------------------------------------------------------------===// -SDValue DAGTypeLegalizer::ExpandOp_BIT_CONVERT(SDNode *N) { +void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements, + SmallVectorImpl &Ops, + EVT EltVT) { + assert(Op.getValueType().isInteger()); + SDLoc DL(Op); + SDValue Parts[2]; + + if (NumElements > 1) { + NumElements >>= 1; + SplitInteger(Op, Parts[0], Parts[1]); + if (TLI.isBigEndian()) + std::swap(Parts[0], Parts[1]); + IntegerToVector(Parts[0], NumElements, Ops, EltVT); + IntegerToVector(Parts[1], NumElements, Ops, EltVT); + } else { + Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op)); + } +} + +SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) { + SDLoc dl(N); if (N->getValueType(0).isVector()) { // An illegal expanding type is being converted to a legal vector type. // Make a two element vector out of the expanded parts and convert that // instead, but only if the new vector type is legal (otherwise there // is no point, and it might create expansion loops). For example, on - // x86 this turns v1i64 = BIT_CONVERT i64 into v1i64 = BIT_CONVERT v2i32. - MVT OVT = N->getOperand(0).getValueType(); - MVT NVT = MVT::getVectorVT(TLI.getTypeToTransformTo(OVT), 2); + // x86 this turns v1i64 = BITCAST i64 into v1i64 = BITCAST v2i32. + // + // FIXME: I'm not sure why we are first trying to split the input into + // a 2 element vector, so I'm leaving it here to maintain the current + // behavior. + unsigned NumElts = 2; + EVT OVT = N->getOperand(0).getValueType(); + EVT NVT = EVT::getVectorVT(*DAG.getContext(), + TLI.getTypeToTransformTo(*DAG.getContext(), OVT), + NumElts); + if (!isTypeLegal(NVT)) { + // If we can't find a legal type by splitting the integer in half, + // then we can use the node's value type. + NumElts = N->getValueType(0).getVectorNumElements(); + NVT = N->getValueType(0); + } - if (isTypeLegal(NVT)) { - SDValue Parts[2]; - GetExpandedOp(N->getOperand(0), Parts[0], Parts[1]); + SmallVector Ops; + IntegerToVector(N->getOperand(0), NumElts, Ops, NVT.getVectorElementType()); - if (TLI.isBigEndian()) - std::swap(Parts[0], Parts[1]); - - SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, NVT, Parts, 2); - return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), Vec); - } + SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, + makeArrayRef(Ops.data(), NumElts)); + return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec); } // Otherwise, store to a temporary and load out again as the new type. @@ -203,10 +372,14 @@ SDValue DAGTypeLegalizer::ExpandOp_BIT_CONVERT(SDNode *N) { SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) { // The vector type is legal but the element type needs expansion. - MVT VecVT = N->getValueType(0); + EVT VecVT = N->getValueType(0); unsigned NumElts = VecVT.getVectorNumElements(); - MVT OldVT = N->getOperand(0).getValueType(); - MVT NewVT = TLI.getTypeToTransformTo(OldVT); + EVT OldVT = N->getOperand(0).getValueType(); + EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); + SDLoc dl(N); + + assert(OldVT == VecVT.getVectorElementType() && + "BUILD_VECTOR operand type doesn't match vector element type!"); // Build a vector of twice the length out of the expanded elements. // For example <3 x i64> -> <6 x i32>. @@ -222,31 +395,85 @@ SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) { NewElts.push_back(Hi); } - SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, - MVT::getVectorVT(NewVT, NewElts.size()), - &NewElts[0], NewElts.size()); + SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, + EVT::getVectorVT(*DAG.getContext(), + NewVT, NewElts.size()), + NewElts); // Convert the new vector to the old vector type. - return DAG.getNode(ISD::BIT_CONVERT, VecVT, NewVec); + return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); } SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) { SDValue Lo, Hi; GetExpandedOp(N->getOperand(0), Lo, Hi); - return cast(N->getOperand(1))->getValue() ? Hi : Lo; + return cast(N->getOperand(1))->getZExtValue() ? Hi : Lo; +} + +SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) { + // The vector type is legal but the element type needs expansion. + EVT VecVT = N->getValueType(0); + unsigned NumElts = VecVT.getVectorNumElements(); + SDLoc dl(N); + + SDValue Val = N->getOperand(1); + EVT OldEVT = Val.getValueType(); + EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT); + + assert(OldEVT == VecVT.getVectorElementType() && + "Inserted element type doesn't match vector element type!"); + + // Bitconvert to a vector of twice the length with elements of the expanded + // type, insert the expanded vector elements, and then convert back. + EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2); + SDValue NewVec = DAG.getNode(ISD::BITCAST, dl, + NewVecVT, N->getOperand(0)); + + SDValue Lo, Hi; + GetExpandedOp(Val, Lo, Hi); + if (TLI.isBigEndian()) + std::swap(Lo, Hi); + + SDValue Idx = N->getOperand(2); + Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); + NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx); + Idx = DAG.getNode(ISD::ADD, dl, + Idx.getValueType(), Idx, + DAG.getConstant(1, dl, Idx.getValueType())); + NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx); + + // Convert the new vector to the old vector type. + return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); +} + +SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) { + SDLoc dl(N); + EVT VT = N->getValueType(0); + assert(VT.getVectorElementType() == N->getOperand(0).getValueType() && + "SCALAR_TO_VECTOR operand type doesn't match vector element type!"); + unsigned NumElts = VT.getVectorNumElements(); + SmallVector Ops(NumElts); + Ops[0] = N->getOperand(0); + SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType()); + for (unsigned i = 1; i < NumElts; ++i) + Ops[i] = UndefVal; + return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); } SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) { assert(ISD::isNormalStore(N) && "This routine only for normal stores!"); assert(OpNo == 1 && "Can only expand the stored value so far"); + SDLoc dl(N); StoreSDNode *St = cast(N); - MVT NVT = TLI.getTypeToTransformTo(St->getValue().getValueType()); + EVT ValueVT = St->getValue().getValueType(); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT); SDValue Chain = St->getChain(); SDValue Ptr = St->getBasePtr(); - int SVOffset = St->getSrcValueOffset(); unsigned Alignment = St->getAlignment(); bool isVolatile = St->isVolatile(); + bool isNonTemporal = St->isNonTemporal(); + AAMDNodes AAInfo = St->getAAInfo(); assert(NVT.isByteSized() && "Expanded type not byte sized!"); unsigned IncrementSize = NVT.getSizeInBits() / 8; @@ -254,19 +481,20 @@ SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) { SDValue Lo, Hi; GetExpandedOp(St->getValue(), Lo, Hi); - if (TLI.isBigEndian()) + if (TLI.hasBigEndianPartOrdering(ValueVT)) std::swap(Lo, Hi); - Lo = DAG.getStore(Chain, Lo, Ptr, St->getSrcValue(), SVOffset, - isVolatile, Alignment); + Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(), + isVolatile, isNonTemporal, Alignment, AAInfo); - Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, - DAG.getIntPtrConstant(IncrementSize)); - assert(isTypeLegal(Ptr.getValueType()) && "Pointers must be legal!"); - Hi = DAG.getStore(Chain, Hi, Ptr, St->getSrcValue(), SVOffset + IncrementSize, - isVolatile, MinAlign(Alignment, IncrementSize)); + Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, + DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); + Hi = DAG.getStore(Chain, dl, Hi, Ptr, + St->getPointerInfo().getWithOffset(IncrementSize), + isVolatile, isNonTemporal, + MinAlign(Alignment, IncrementSize), AAInfo); - return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); + return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); } @@ -279,53 +507,50 @@ SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) { // bytes; for integers and floats it is Lo first if and only if the machine is // little-endian). -void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, +void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo, SDValue &Lo, SDValue &Hi) { - // A MERGE_VALUES node can produce any number of values. We know that the - // first illegal one needs to be expanded into Lo/Hi. - unsigned i; - - // The string of legal results gets turns into the input operands, which have - // the same type. - for (i = 0; isTypeLegal(N->getValueType(i)); ++i) - ReplaceValueWith(SDValue(N, i), SDValue(N->getOperand(i))); - - // The first illegal result must be the one that needs to be expanded. - GetSplitOp(N->getOperand(i), Lo, Hi); - - // Legalize the rest of the results into the input operands whether they are - // legal or not. - unsigned e = N->getNumValues(); - for (++i; i != e; ++i) - ReplaceValueWith(SDValue(N, i), SDValue(N->getOperand(i))); + SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); + GetSplitOp(Op, Lo, Hi); } void DAGTypeLegalizer::SplitRes_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi) { - SDValue LL, LH, RL, RH; + SDValue LL, LH, RL, RH, CL, CH; + SDLoc dl(N); GetSplitOp(N->getOperand(1), LL, LH); GetSplitOp(N->getOperand(2), RL, RH); SDValue Cond = N->getOperand(0); - Lo = DAG.getNode(ISD::SELECT, LL.getValueType(), Cond, LL, RL); - Hi = DAG.getNode(ISD::SELECT, LH.getValueType(), Cond, LH, RH); + CL = CH = Cond; + if (Cond.getValueType().isVector()) { + // Check if there are already splitted versions of the vector available and + // use those instead of splitting the mask operand again. + if (getTypeAction(Cond.getValueType()) == TargetLowering::TypeSplitVector) + GetSplitVector(Cond, CL, CH); + else + std::tie(CL, CH) = DAG.SplitVector(Cond, dl); + } + + Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), CL, LL, RL); + Hi = DAG.getNode(N->getOpcode(), dl, LH.getValueType(), CH, LH, RH); } void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue LL, LH, RL, RH; + SDLoc dl(N); GetSplitOp(N->getOperand(2), LL, LH); GetSplitOp(N->getOperand(3), RL, RH); - Lo = DAG.getNode(ISD::SELECT_CC, LL.getValueType(), N->getOperand(0), + Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0), N->getOperand(1), LL, RL, N->getOperand(4)); - Hi = DAG.getNode(ISD::SELECT_CC, LH.getValueType(), N->getOperand(0), + Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0), N->getOperand(1), LH, RH, N->getOperand(4)); } void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) { - MVT LoVT, HiVT; - GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); - Lo = DAG.getNode(ISD::UNDEF, LoVT); - Hi = DAG.getNode(ISD::UNDEF, HiVT); + EVT LoVT, HiVT; + std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); + Lo = DAG.getUNDEF(LoVT); + Hi = DAG.getUNDEF(HiVT); }