From cd6e725f21852e2f8cdf5fd0e65eb42c224776f8 Mon Sep 17 00:00:00 2001 From: Mon P Wang Date: Mon, 30 Nov 2009 02:42:02 +0000 Subject: [PATCH] Added support to allow clients to custom widen. For X86, custom widen vectors for divide/remainder since these operations can trap by unroll them and adding undefs for the resulting vector. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@90108 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/CodeGen/SelectionDAG.h | 8 +++ lib/CodeGen/SelectionDAG/LegalizeTypes.cpp | 23 +++++++ lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 + .../SelectionDAG/LegalizeVectorOps.cpp | 57 +---------------- .../SelectionDAG/LegalizeVectorTypes.cpp | 6 +- lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 61 +++++++++++++++++++ lib/Target/X86/X86ISelLowering.cpp | 21 +++++++ 7 files changed, 121 insertions(+), 56 deletions(-) diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index f194e4e3f96..e5868079cda 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -882,6 +882,14 @@ public: /// element of the result of the vector shuffle. SDValue getShuffleScalarElt(const ShuffleVectorSDNode *N, unsigned Idx); + /// UnrollVectorOp - Utility function used by legalize and lowering to + /// "unroll" a vector operation by splitting out the scalars and operating + /// on each element individually. If the ResNE is 0, fully unroll the vector + /// op. If ResNE is less than the width of the vector op, unroll up to ResNE. + /// If the ResNE is greater than the width of the vector op, unroll the + /// vector op and fill the end of the resulting vector with UNDEFS. + SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0); + private: bool RemoveNodeFromCSEMaps(SDNode *N); void AddModifiedNodeToCSEMaps(SDNode *N, DAGUpdateListener *UpdateListener); diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp index e2986493d4d..003cea7a6f4 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp @@ -907,6 +907,29 @@ bool DAGTypeLegalizer::CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult) { return true; } + +/// CustomWidenLowerNode - Widen the node's results with custom code provided +/// by the target and return "true", or do nothing and return "false". +bool DAGTypeLegalizer::CustomWidenLowerNode(SDNode *N, EVT VT) { + // See if the target wants to custom lower this node. + if (TLI.getOperationAction(N->getOpcode(), VT) != TargetLowering::Custom) + return false; + + SmallVector Results; + TLI.ReplaceNodeResults(N, Results, DAG); + + if (Results.empty()) + // The target didn't want to custom widen lower its result after all. + return false; + + // Update the widening map. + assert(Results.size() == N->getNumValues() && + "Custom lowering returned the wrong number of results!"); + for (unsigned i = 0, e = Results.size(); i != e; ++i) + SetWidenedVector(SDValue(N, i), Results[i]); + return true; +} + /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type /// which is split into two not necessarily identical pieces. void DAGTypeLegalizer::GetSplitDestVTs(EVT InVT, EVT &LoVT, EVT &HiVT) { diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 7b9b010e630..2ee9f8a42ed 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -188,6 +188,7 @@ private: SDValue BitConvertVectorToIntegerVector(SDValue Op); SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT); bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult); + bool CustomWidenLowerNode(SDNode *N, EVT VT); SDValue GetVectorElementPointer(SDValue VecPtr, EVT EltVT, SDValue Index); SDValue JoinIntegers(SDValue Lo, SDValue Hi); SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned); diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index ca194305d98..785c2adb394 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -54,9 +54,6 @@ class VectorLegalizer { SDValue LegalizeOp(SDValue Op); // Assuming the node is legal, "legalize" the results SDValue TranslateLegalizeResults(SDValue Op, SDValue Result); - // Implements unrolling a generic vector operation, i.e. turning it into - // scalar operations. - SDValue UnrollVectorOp(SDValue Op); // Implements unrolling a VSETCC. SDValue UnrollVSETCC(SDValue Op); // Implements expansion for FNEG; falls back to UnrollVectorOp if FSUB @@ -211,7 +208,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { else if (Node->getOpcode() == ISD::VSETCC) Result = UnrollVSETCC(Op); else - Result = UnrollVectorOp(Op); + Result = DAG.UnrollVectorOp(Op.getNode()); break; } @@ -256,7 +253,7 @@ SDValue VectorLegalizer::ExpandFNEG(SDValue Op) { return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(), Zero, Op.getOperand(0)); } - return UnrollVectorOp(Op); + return DAG.UnrollVectorOp(Op.getNode()); } SDValue VectorLegalizer::UnrollVSETCC(SDValue Op) { @@ -282,56 +279,6 @@ SDValue VectorLegalizer::UnrollVSETCC(SDValue Op) { return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElems); } -/// UnrollVectorOp - We know that the given vector has a legal type, however -/// the operation it performs is not legal, and the target has requested that -/// the operation be expanded. "Unroll" the vector, splitting out the scalars -/// and operating on each element individually. -SDValue VectorLegalizer::UnrollVectorOp(SDValue Op) { - EVT VT = Op.getValueType(); - assert(Op.getNode()->getNumValues() == 1 && - "Can't unroll a vector with multiple results!"); - unsigned NE = VT.getVectorNumElements(); - EVT EltVT = VT.getVectorElementType(); - DebugLoc dl = Op.getDebugLoc(); - - SmallVector Scalars; - SmallVector Operands(Op.getNumOperands()); - for (unsigned i = 0; i != NE; ++i) { - for (unsigned j = 0; j != Op.getNumOperands(); ++j) { - SDValue Operand = Op.getOperand(j); - EVT OperandVT = Operand.getValueType(); - if (OperandVT.isVector()) { - // A vector operand; extract a single element. - EVT OperandEltVT = OperandVT.getVectorElementType(); - Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, - OperandEltVT, - Operand, - DAG.getConstant(i, MVT::i32)); - } else { - // A scalar operand; just use it as is. - Operands[j] = Operand; - } - } - - switch (Op.getOpcode()) { - default: - Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT, - &Operands[0], Operands.size())); - break; - case ISD::SHL: - case ISD::SRA: - case ISD::SRL: - case ISD::ROTL: - case ISD::ROTR: - Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT, Operands[0], - DAG.getShiftAmountOperand(Operands[1]))); - break; - } - } - - return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Scalars[0], Scalars.size()); -} - } bool SelectionDAG::LegalizeVectors() { diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 75e12395d8b..023324b82af 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -1118,8 +1118,12 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { DEBUG(errs() << "Widen node result " << ResNo << ": "; N->dump(&DAG); errs() << "\n"); - SDValue Res = SDValue(); + // See if the target wants to custom widen this node. + if (CustomWidenLowerNode(N, N->getValueType(ResNo))) + return; + + SDValue Res = SDValue(); switch (N->getOpcode()) { default: #ifndef NDEBUG diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 8f999570842..c38c79b1459 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5807,6 +5807,66 @@ static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) { N->dump(G); } +SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { + assert(N->getNumValues() == 1 && + "Can't unroll a vector with multiple results!"); + + EVT VT = N->getValueType(0); + unsigned NE = VT.getVectorNumElements(); + EVT EltVT = VT.getVectorElementType(); + DebugLoc dl = N->getDebugLoc(); + + SmallVector Scalars; + SmallVector Operands(N->getNumOperands()); + + // If ResNE is 0, fully unroll the vector op. + if (ResNE == 0) + ResNE = NE; + else if (NE > ResNE) + NE = ResNE; + + unsigned i; + for (i= 0; i != NE; ++i) { + for (unsigned j = 0; j != N->getNumOperands(); ++j) { + SDValue Operand = N->getOperand(j); + EVT OperandVT = Operand.getValueType(); + if (OperandVT.isVector()) { + // A vector operand; extract a single element. + EVT OperandEltVT = OperandVT.getVectorElementType(); + Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, + OperandEltVT, + Operand, + getConstant(i, MVT::i32)); + } else { + // A scalar operand; just use it as is. + Operands[j] = Operand; + } + } + + switch (N->getOpcode()) { + default: + Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, + &Operands[0], Operands.size())); + break; + case ISD::SHL: + case ISD::SRA: + case ISD::SRL: + case ISD::ROTL: + case ISD::ROTR: + Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], + getShiftAmountOperand(Operands[1]))); + break; + } + } + + for (; i < ResNE; ++i) + Scalars.push_back(getUNDEF(EltVT)); + + return getNode(ISD::BUILD_VECTOR, dl, + EVT::getVectorVT(*getContext(), EltVT, ResNE), + &Scalars[0], Scalars.size()); +} + void SelectionDAG::dump() const { errs() << "SelectionDAG has " << AllNodes.size() << " nodes:"; @@ -5962,3 +6022,4 @@ bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { return false; return true; } + diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index 8567ca47b7c..d80b8ec66ec 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -975,6 +975,19 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM) computeRegisterProperties(); + // Divide and reminder operations have no vector equivalent and can + // trap. Do a custom widening for these operations in which we never + // generate more divides/remainder than the original vector width. + for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; + VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) { + if (!isTypeLegal((MVT::SimpleValueType)VT)) { + setOperationAction(ISD::SDIV, (MVT::SimpleValueType) VT, Custom); + setOperationAction(ISD::UDIV, (MVT::SimpleValueType) VT, Custom); + setOperationAction(ISD::SREM, (MVT::SimpleValueType) VT, Custom); + setOperationAction(ISD::UREM, (MVT::SimpleValueType) VT, Custom); + } + } + // FIXME: These should be based on subtarget info. Plus, the values should // be smaller when we are in optimizing for size mode. maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores @@ -7170,6 +7183,14 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, Results.push_back(edx.getValue(1)); return; } + case ISD::SDIV: + case ISD::UDIV: + case ISD::SREM: + case ISD::UREM: { + EVT WidenVT = getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + Results.push_back(DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements())); + return; + } case ISD::ATOMIC_CMP_SWAP: { EVT T = N->getValueType(0); assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap"); -- 2.34.1