From: Dan Gohman Date: Mon, 8 Oct 2007 18:33:35 +0000 (+0000) Subject: Migrate X86 and ARM from using X86ISD::{,I}DIV and ARMISD::MULHILO{U,S} to X-Git-Url: http://plrg.eecs.uci.edu/git/?a=commitdiff_plain;h=525178cdbf00720ea8bce297a7d65b0cca0ab439;p=oota-llvm.git Migrate X86 and ARM from using X86ISD::{,I}DIV and ARMISD::MULHILO{U,S} to use ISD::{S,U}DIVREM and ISD::{S,U}MUL_HIO. Move the lowering code associated with these operators into target-independent in LegalizeDAG.cpp and TargetLowering.cpp. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@42762 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 09f45d2ce7e..98a2495d28d 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -2577,6 +2577,55 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { if (Tmp1.Val) Result = Tmp1; break; case TargetLowering::Expand: { + MVT::ValueType VT = Op.getValueType(); + + // See if multiply or divide can be lowered using two-result operations. + SDVTList VTs = DAG.getVTList(VT, VT); + if (Node->getOpcode() == ISD::MUL) { + // We just need the low half of the multiply; try both the signed + // and unsigned forms. If the target supports both SMUL_LOHI and + // UMUL_LOHI, form a preference by checking which forms of plain + // MULH it supports. + bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, VT); + bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, VT); + bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, VT); + bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, VT); + unsigned OpToUse = 0; + if (HasSMUL_LOHI && !HasMULHS) { + OpToUse = ISD::SMUL_LOHI; + } else if (HasUMUL_LOHI && !HasMULHU) { + OpToUse = ISD::UMUL_LOHI; + } else if (HasSMUL_LOHI) { + OpToUse = ISD::SMUL_LOHI; + } else if (HasUMUL_LOHI) { + OpToUse = ISD::UMUL_LOHI; + } + if (OpToUse) { + Result = SDOperand(DAG.getNode(OpToUse, VTs, Tmp1, Tmp2).Val, 0); + break; + } + } + if (Node->getOpcode() == ISD::MULHS && + TLI.isOperationLegal(ISD::SMUL_LOHI, VT)) { + Result = SDOperand(DAG.getNode(ISD::SMUL_LOHI, VTs, Tmp1, Tmp2).Val, 1); + break; + } + if (Node->getOpcode() == ISD::MULHU && + TLI.isOperationLegal(ISD::UMUL_LOHI, VT)) { + Result = SDOperand(DAG.getNode(ISD::UMUL_LOHI, VTs, Tmp1, Tmp2).Val, 1); + break; + } + if (Node->getOpcode() == ISD::SDIV && + TLI.isOperationLegal(ISD::SDIVREM, VT)) { + Result = SDOperand(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).Val, 0); + break; + } + if (Node->getOpcode() == ISD::UDIV && + TLI.isOperationLegal(ISD::UDIVREM, VT)) { + Result = SDOperand(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).Val, 0); + break; + } + if (Node->getValueType(0) == MVT::i32) { switch (Node->getOpcode()) { default: assert(0 && "Do not know how to expand this integer BinOp!"); @@ -2638,6 +2687,10 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // they shouldn't be here if they aren't legal. assert(TLI.isOperationLegal(Node->getValueType(0), Node->getValueType(0)) && "This must be legal!"); + + Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS + Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS + Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); break; case ISD::FCOPYSIGN: // FCOPYSIGN does not require LHS/RHS to match type! @@ -2764,19 +2817,33 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { if (Tmp1.Val) Result = Tmp1; } break; - case TargetLowering::Expand: + case TargetLowering::Expand: { unsigned DivOpc= (Node->getOpcode() == ISD::UREM) ? ISD::UDIV : ISD::SDIV; bool isSigned = DivOpc == ISD::SDIV; - if (MVT::isInteger(Node->getValueType(0))) { - if (TLI.getOperationAction(DivOpc, Node->getValueType(0)) == + MVT::ValueType VT = Node->getValueType(0); + + // See if remainder can be lowered using two-result operations. + SDVTList VTs = DAG.getVTList(VT, VT); + if (Node->getOpcode() == ISD::SREM && + TLI.isOperationLegal(ISD::SDIVREM, VT)) { + Result = SDOperand(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).Val, 1); + break; + } + if (Node->getOpcode() == ISD::UREM && + TLI.isOperationLegal(ISD::UDIVREM, VT)) { + Result = SDOperand(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).Val, 1); + break; + } + + if (MVT::isInteger(VT)) { + if (TLI.getOperationAction(DivOpc, VT) == TargetLowering::Legal) { // X % Y -> X-X/Y*Y - MVT::ValueType VT = Node->getValueType(0); Result = DAG.getNode(DivOpc, VT, Tmp1, Tmp2); Result = DAG.getNode(ISD::MUL, VT, Result, Tmp2); Result = DAG.getNode(ISD::SUB, VT, Tmp1, Result); } else { - assert(Node->getValueType(0) == MVT::i32 && + assert(VT == MVT::i32 && "Cannot expand this binary operator!"); RTLIB::Libcall LC = Node->getOpcode() == ISD::UREM ? RTLIB::UREM_I32 : RTLIB::SREM_I32; @@ -2785,7 +2852,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } } else { // Floating point mod -> fmod libcall. - RTLIB::Libcall LC = Node->getValueType(0) == MVT::f32 + RTLIB::Libcall LC = VT == MVT::f32 ? RTLIB::REM_F32 : RTLIB::REM_F64; SDOperand Dummy; Result = ExpandLibCall(TLI.getLibcallName(LC), Node, @@ -2793,6 +2860,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } break; } + } break; case ISD::VAARG: { Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. @@ -5666,36 +5734,55 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT); bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT); - if (HasMULHS || HasMULHU) { + bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT); + bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT); + if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { SDOperand LL, LH, RL, RH; ExpandOp(Node->getOperand(0), LL, LH); ExpandOp(Node->getOperand(1), RL, RH); - unsigned SH = MVT::getSizeInBits(RH.getValueType())-1; - // FIXME: Move this to the dag combiner. - // MULHS implicitly sign extends its inputs. Check to see if ExpandOp - // extended the sign bit of the low half through the upper half, and if so - // emit a MULHS instead of the alternate sequence that is valid for any - // i64 x i64 multiply. - if (HasMULHS && - // is RH an extension of the sign bit of RL? - RH.getOpcode() == ISD::SRA && RH.getOperand(0) == RL && - RH.getOperand(1).getOpcode() == ISD::Constant && - cast(RH.getOperand(1))->getValue() == SH && - // is LH an extension of the sign bit of LL? - LH.getOpcode() == ISD::SRA && LH.getOperand(0) == LL && - LH.getOperand(1).getOpcode() == ISD::Constant && - cast(LH.getOperand(1))->getValue() == SH) { - // Low part: - Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); - // High part: - Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL); + unsigned BitSize = MVT::getSizeInBits(RH.getValueType()); + unsigned LHSSB = DAG.ComputeNumSignBits(Op.getOperand(0)); + unsigned RHSSB = DAG.ComputeNumSignBits(Op.getOperand(1)); + // FIXME: generalize this to handle other bit sizes + if (LHSSB == 32 && RHSSB == 32 && + DAG.MaskedValueIsZero(Op.getOperand(0), 0xFFFFFFFF00000000ULL) && + DAG.MaskedValueIsZero(Op.getOperand(1), 0xFFFFFFFF00000000ULL)) { + // The inputs are both zero-extended. + if (HasUMUL_LOHI) { + // We can emit a umul_lohi. + Lo = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); + Hi = SDOperand(Lo.Val, 1); + break; + } + if (HasMULHU) { + // We can emit a mulhu+mul. + Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); + Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL); + break; + } break; - } else if (HasMULHU) { - // Low part: - Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); - - // High part: - Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL); + } + if (LHSSB > BitSize && RHSSB > BitSize) { + // The input values are both sign-extended. + if (HasSMUL_LOHI) { + // We can emit a smul_lohi. + Lo = DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); + Hi = SDOperand(Lo.Val, 1); + break; + } + if (HasMULHS) { + // We can emit a mulhs+mul. + Lo = DAG.getNode(ISD::MUL, NVT, LL, RL); + Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL); + break; + } + } + if (HasUMUL_LOHI) { + // Lo,Hi = umul LHS, RHS. + SDOperand UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, + DAG.getVTList(NVT, NVT), LL, RL); + Lo = UMulLOHI; + Hi = UMulLOHI.getValue(1); RH = DAG.getNode(ISD::MUL, NVT, LL, RH); LH = DAG.getNode(ISD::MUL, NVT, LH, RL); Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH); @@ -5704,6 +5791,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ } } + // If nothing else, we can make a libcall. Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::MUL_I64), Node, false/*sign irrelevant*/, Hi); break; diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index e00f2f08e4b..d55b7452ee5 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -1706,15 +1706,21 @@ SDOperand TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG, // Check to see if we can do this. if (!isTypeLegal(VT) || (VT != MVT::i32 && VT != MVT::i64)) return SDOperand(); // BuildSDIV only operates on i32 or i64 - if (!isOperationLegal(ISD::MULHS, VT)) - return SDOperand(); // Make sure the target supports MULHS. int64_t d = cast(N->getOperand(1))->getSignExtended(); ms magics = (VT == MVT::i32) ? magic32(d) : magic64(d); // Multiply the numerator (operand 0) by the magic value - SDOperand Q = DAG.getNode(ISD::MULHS, VT, N->getOperand(0), - DAG.getConstant(magics.m, VT)); + SDOperand Q; + if (isOperationLegal(ISD::MULHS, VT)) + Q = DAG.getNode(ISD::MULHS, VT, N->getOperand(0), + DAG.getConstant(magics.m, VT)); + else if (isOperationLegal(ISD::SMUL_LOHI, VT)) + Q = SDOperand(DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(VT, VT), + N->getOperand(0), + DAG.getConstant(magics.m, VT)).Val, 1); + else + return SDOperand(); // No mulhs or equvialent // If d > 0 and m < 0, add the numerator if (d > 0 && magics.m < 0) { Q = DAG.getNode(ISD::ADD, VT, Q, N->getOperand(0)); @@ -1754,15 +1760,21 @@ SDOperand TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG, // Check to see if we can do this. if (!isTypeLegal(VT) || (VT != MVT::i32 && VT != MVT::i64)) return SDOperand(); // BuildUDIV only operates on i32 or i64 - if (!isOperationLegal(ISD::MULHU, VT)) - return SDOperand(); // Make sure the target supports MULHU. uint64_t d = cast(N->getOperand(1))->getValue(); mu magics = (VT == MVT::i32) ? magicu32(d) : magicu64(d); // Multiply the numerator (operand 0) by the magic value - SDOperand Q = DAG.getNode(ISD::MULHU, VT, N->getOperand(0), - DAG.getConstant(magics.m, VT)); + SDOperand Q; + if (isOperationLegal(ISD::MULHU, VT)) + Q = DAG.getNode(ISD::MULHU, VT, N->getOperand(0), + DAG.getConstant(magics.m, VT)); + else if (isOperationLegal(ISD::UMUL_LOHI, VT)) + Q = SDOperand(DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(VT, VT), + N->getOperand(0), + DAG.getConstant(magics.m, VT)).Val, 1); + else + return SDOperand(); // No mulhu or equvialent if (Created) Created->push_back(Q.Val); diff --git a/lib/Target/ARM/ARMISelDAGToDAG.cpp b/lib/Target/ARM/ARMISelDAGToDAG.cpp index 933651d8b15..a779eafde81 100644 --- a/lib/Target/ARM/ARMISelDAGToDAG.cpp +++ b/lib/Target/ARM/ARMISelDAGToDAG.cpp @@ -641,7 +641,7 @@ SDNode *ARMDAGToDAGISel::Select(SDOperand Op) { return CurDAG->getTargetNode(ARM::FMRRD, MVT::i32, MVT::i32, Op.getOperand(0), getAL(CurDAG), CurDAG->getRegister(0, MVT::i32)); - case ARMISD::MULHILOU: { + case ISD::UMUL_LOHI: { AddToISelQueue(Op.getOperand(0)); AddToISelQueue(Op.getOperand(1)); SDOperand Ops[] = { Op.getOperand(0), Op.getOperand(1), @@ -649,7 +649,7 @@ SDNode *ARMDAGToDAGISel::Select(SDOperand Op) { CurDAG->getRegister(0, MVT::i32) }; return CurDAG->getTargetNode(ARM::UMULL, MVT::i32, MVT::i32, Ops, 5); } - case ARMISD::MULHILOS: { + case ISD::SMUL_LOHI: { AddToISelQueue(Op.getOperand(0)); AddToISelQueue(Op.getOperand(1)); SDOperand Ops[] = { Op.getOperand(0), Op.getOperand(1), diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp index c41bb2ee463..0c43e6dccd2 100644 --- a/lib/Target/ARM/ARMISelLowering.cpp +++ b/lib/Target/ARM/ARMISelLowering.cpp @@ -147,11 +147,13 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM) setOperationAction(ISD::MUL, MVT::i64, Expand); setOperationAction(ISD::MULHU, MVT::i32, Expand); setOperationAction(ISD::MULHS, MVT::i32, Expand); + setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); + setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); } else { - setOperationAction(ISD::MUL, MVT::i64, Custom); - setOperationAction(ISD::MULHU, MVT::i32, Custom); + setOperationAction(ISD::MUL, MVT::i64, Expand); + setOperationAction(ISD::MULHU, MVT::i32, Expand); if (!Subtarget->hasV6Ops()) - setOperationAction(ISD::MULHS, MVT::i32, Custom); + setOperationAction(ISD::MULHS, MVT::i32, Expand); } setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand); setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand); @@ -175,6 +177,8 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM) setOperationAction(ISD::UDIV, MVT::i32, Expand); setOperationAction(ISD::SREM, MVT::i32, Expand); setOperationAction(ISD::UREM, MVT::i32, Expand); + setOperationAction(ISD::SDIVREM, MVT::i32, Expand); + setOperationAction(ISD::UDIVREM, MVT::i32, Expand); // Support label based line numbers. setOperationAction(ISD::LOCATION, MVT::Other, Expand); @@ -286,8 +290,6 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const { case ARMISD::FTOUI: return "ARMISD::FTOUI"; case ARMISD::SITOF: return "ARMISD::SITOF"; case ARMISD::UITOF: return "ARMISD::UITOF"; - case ARMISD::MULHILOU: return "ARMISD::MULHILOU"; - case ARMISD::MULHILOS: return "ARMISD::MULHILOS"; case ARMISD::SRL_FLAG: return "ARMISD::SRL_FLAG"; case ARMISD::SRA_FLAG: return "ARMISD::SRA_FLAG"; @@ -1249,66 +1251,6 @@ static SDOperand LowerBIT_CONVERT(SDOperand Op, SelectionDAG &DAG) { return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Cvt, Cvt.getValue(1)); } -static SDOperand LowerMUL(SDOperand Op, SelectionDAG &DAG) { - // FIXME: All this code is target-independent. Create a new target-indep - // MULHILO node and move this code to the legalizer. - // - assert(Op.getValueType() == MVT::i64 && "Only handles i64 expand right now!"); - - SDOperand LL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op.getOperand(0), - DAG.getConstant(0, MVT::i32)); - SDOperand RL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op.getOperand(1), - DAG.getConstant(0, MVT::i32)); - - unsigned LHSSB = DAG.ComputeNumSignBits(Op.getOperand(0)); - unsigned RHSSB = DAG.ComputeNumSignBits(Op.getOperand(1)); - - SDOperand Lo, Hi; - // Figure out how to lower this multiply. - if (LHSSB >= 33 && RHSSB >= 33) { - // If the input values are both sign extended, we can emit a mulhs+mul. - Lo = DAG.getNode(ISD::MUL, MVT::i32, LL, RL); - Hi = DAG.getNode(ISD::MULHS, MVT::i32, LL, RL); - } else if (LHSSB == 32 && RHSSB == 32 && - DAG.MaskedValueIsZero(Op.getOperand(0), 0xFFFFFFFF00000000ULL) && - DAG.MaskedValueIsZero(Op.getOperand(1), 0xFFFFFFFF00000000ULL)) { - // If the inputs are zero extended, use mulhu. - Lo = DAG.getNode(ISD::MUL, MVT::i32, LL, RL); - Hi = DAG.getNode(ISD::MULHU, MVT::i32, LL, RL); - } else { - SDOperand LH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op.getOperand(0), - DAG.getConstant(1, MVT::i32)); - SDOperand RH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op.getOperand(1), - DAG.getConstant(1, MVT::i32)); - - // Lo,Hi = umul LHS, RHS. - SDOperand Ops[] = { LL, RL }; - SDOperand UMul64 = DAG.getNode(ARMISD::MULHILOU, - DAG.getVTList(MVT::i32, MVT::i32), Ops, 2); - Lo = UMul64; - Hi = UMul64.getValue(1); - RH = DAG.getNode(ISD::MUL, MVT::i32, LL, RH); - LH = DAG.getNode(ISD::MUL, MVT::i32, LH, RL); - Hi = DAG.getNode(ISD::ADD, MVT::i32, Hi, RH); - Hi = DAG.getNode(ISD::ADD, MVT::i32, Hi, LH); - } - - // Merge the pieces into a single i64 value. - return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi); -} - -static SDOperand LowerMULHU(SDOperand Op, SelectionDAG &DAG) { - SDOperand Ops[] = { Op.getOperand(0), Op.getOperand(1) }; - return DAG.getNode(ARMISD::MULHILOU, - DAG.getVTList(MVT::i32, MVT::i32), Ops, 2).getValue(1); -} - -static SDOperand LowerMULHS(SDOperand Op, SelectionDAG &DAG) { - SDOperand Ops[] = { Op.getOperand(0), Op.getOperand(1) }; - return DAG.getNode(ARMISD::MULHILOS, - DAG.getVTList(MVT::i32, MVT::i32), Ops, 2).getValue(1); -} - static SDOperand LowerSRx(SDOperand Op, SelectionDAG &DAG, const ARMSubtarget *ST) { assert(Op.getValueType() == MVT::i64 && @@ -1433,9 +1375,6 @@ SDOperand ARMTargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG); case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); case ISD::BIT_CONVERT: return LowerBIT_CONVERT(Op, DAG); - case ISD::MUL: return LowerMUL(Op, DAG); - case ISD::MULHU: return LowerMULHU(Op, DAG); - case ISD::MULHS: return LowerMULHS(Op, DAG); case ISD::SRL: case ISD::SRA: return LowerSRx(Op, DAG, Subtarget); case ISD::FORMAL_ARGUMENTS: diff --git a/lib/Target/ARM/ARMISelLowering.h b/lib/Target/ARM/ARMISelLowering.h index ea2ab74375a..b26174f03cb 100644 --- a/lib/Target/ARM/ARMISelLowering.h +++ b/lib/Target/ARM/ARMISelLowering.h @@ -56,9 +56,6 @@ namespace llvm { SITOF, // sint to FP within a FP register. UITOF, // uint to FP within a FP register. - MULHILOU, // Lo,Hi = umul LHS, RHS. - MULHILOS, // Lo,Hi = smul LHS, RHS. - SRL_FLAG, // V,Flag = srl_flag X -> srl X, 1 + save carry out. SRA_FLAG, // V,Flag = sra_flag X -> sra X, 1 + save carry out. RRX, // V = RRX X, Flag -> srl X, 1 + shift in carry flag. diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp index 94803cea27e..6c741768d8c 100644 --- a/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -1085,9 +1085,22 @@ SDNode *X86DAGToDAGISel::Select(SDOperand N) { break; } - case ISD::MULHU: - case ISD::MULHS: { - if (Opcode == ISD::MULHU) + case ISD::SMUL_LOHI: + case ISD::UMUL_LOHI: { + SDOperand N0 = Node->getOperand(0); + SDOperand N1 = Node->getOperand(1); + + // There are several forms of IMUL just return the low part and don't + // have fixed-register operands. If we don't need the high part, use + // these instead. They can be selected with the generated ISel code. + if (NVT != MVT::i8 && + N.getValue(1).use_empty()) { + N = CurDAG->getNode(ISD::MUL, NVT, N0, N1); + break; + } + + bool isSigned = Opcode == ISD::SMUL_LOHI; + if (!isSigned) switch (NVT) { default: assert(0 && "Unsupported VT!"); case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m; break; @@ -1113,78 +1126,90 @@ SDNode *X86DAGToDAGISel::Select(SDOperand N) { case MVT::i64: LoReg = X86::RAX; HiReg = X86::RDX; break; } - SDOperand N0 = Node->getOperand(0); - SDOperand N1 = Node->getOperand(1); - SDOperand Tmp0, Tmp1, Tmp2, Tmp3; bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3); - // MULHU and MULHS are commmutative + // multiplty is commmutative if (!foldedLoad) { foldedLoad = TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3); if (foldedLoad) std::swap(N0, N1); } - SDOperand Chain; - if (foldedLoad) { - Chain = N1.getOperand(0); - AddToISelQueue(Chain); - } else - Chain = CurDAG->getEntryNode(); - - SDOperand InFlag(0, 0); AddToISelQueue(N0); - Chain = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(LoReg, NVT), - N0, InFlag); - InFlag = Chain.getValue(1); + SDOperand InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), LoReg, + N0, SDOperand()).getValue(1); if (foldedLoad) { + AddToISelQueue(N1.getOperand(0)); AddToISelQueue(Tmp0); AddToISelQueue(Tmp1); AddToISelQueue(Tmp2); AddToISelQueue(Tmp3); - SDOperand Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Chain, InFlag }; + SDOperand Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, N1.getOperand(0), InFlag }; SDNode *CNode = CurDAG->getTargetNode(MOpc, MVT::Other, MVT::Flag, Ops, 6); - Chain = SDOperand(CNode, 0); InFlag = SDOperand(CNode, 1); + // Update the chain. + ReplaceUses(N1.getValue(1), SDOperand(CNode, 0)); } else { AddToISelQueue(N1); InFlag = SDOperand(CurDAG->getTargetNode(Opc, MVT::Flag, N1, InFlag), 0); } - SDOperand Result; - if (HiReg == X86::AH && Subtarget->is64Bit()) { - // Prevent use of AH in a REX instruction by referencing AX instead. - // Shift it down 8 bits. - Result = CurDAG->getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag); - Chain = Result.getValue(1); - Result = SDOperand(CurDAG->getTargetNode(X86::SHR16ri, MVT::i16, Result, - CurDAG->getTargetConstant(8, MVT::i8)), 0); - // Then truncate it down to i8. - SDOperand SRIdx = CurDAG->getTargetConstant(1, MVT::i32); // SubRegSet 1 - Result = SDOperand(CurDAG->getTargetNode(X86::EXTRACT_SUBREG, - MVT::i8, Result, SRIdx), 0); - } else { - Result = CurDAG->getCopyFromReg(Chain, HiReg, NVT, InFlag); + // Copy the low half of the result, if it is needed. + if (!N.getValue(0).use_empty()) { + SDOperand Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), + LoReg, NVT, InFlag); + InFlag = Result.getValue(2); + ReplaceUses(N.getValue(0), Result); +#ifndef NDEBUG + DOUT << std::string(Indent-2, ' ') << "=> "; + DEBUG(Result.Val->dump(CurDAG)); + DOUT << "\n"; +#endif + } + // Copy the high half of the result, if it is needed. + if (!N.getValue(1).use_empty()) { + SDOperand Result; + if (HiReg == X86::AH && Subtarget->is64Bit()) { + // Prevent use of AH in a REX instruction by referencing AX instead. + // Shift it down 8 bits. + Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), + X86::AX, MVT::i16, InFlag); + InFlag = Result.getValue(2); + Result = SDOperand(CurDAG->getTargetNode(X86::SHR16ri, MVT::i16, Result, + CurDAG->getTargetConstant(8, MVT::i8)), 0); + // Then truncate it down to i8. + SDOperand SRIdx = CurDAG->getTargetConstant(1, MVT::i32); // SubRegSet 1 + Result = SDOperand(CurDAG->getTargetNode(X86::EXTRACT_SUBREG, + MVT::i8, Result, SRIdx), 0); + } else { + Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), + HiReg, NVT, InFlag); + InFlag = Result.getValue(2); + } + ReplaceUses(N.getValue(1), Result); +#ifndef NDEBUG + DOUT << std::string(Indent-2, ' ') << "=> "; + DEBUG(Result.Val->dump(CurDAG)); + DOUT << "\n"; +#endif } - ReplaceUses(N.getValue(0), Result); - if (foldedLoad) - ReplaceUses(N1.getValue(1), Result.getValue(1)); #ifndef NDEBUG - DOUT << std::string(Indent-2, ' ') << "=> "; - DEBUG(Result.Val->dump(CurDAG)); - DOUT << "\n"; Indent -= 2; #endif + return NULL; } - case X86ISD::DIV: - case X86ISD::IDIV: { - bool isSigned = Opcode == X86ISD::IDIV; + case ISD::SDIVREM: + case ISD::UDIVREM: { + SDOperand N0 = Node->getOperand(0); + SDOperand N1 = Node->getOperand(1); + + bool isSigned = Opcode == ISD::SDIVREM; if (!isSigned) switch (NVT) { default: assert(0 && "Unsupported VT!"); @@ -1228,9 +1253,10 @@ SDNode *X86DAGToDAGISel::Select(SDOperand N) { break; } - SDOperand N0 = Node->getOperand(0); - SDOperand N1 = Node->getOperand(1); - SDOperand InFlag(0, 0); + SDOperand Tmp0, Tmp1, Tmp2, Tmp3; + bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3); + + SDOperand InFlag; if (NVT == MVT::i8 && !isSigned) { // Special case for div8, just use a move with zero extension to AX to // clear the upper 8 bits (AH). @@ -1253,13 +1279,13 @@ SDNode *X86DAGToDAGISel::Select(SDOperand N) { SDOperand(CurDAG->getTargetNode(X86::MOVZX16rr8, MVT::i16, N0), 0); Chain = CurDAG->getEntryNode(); } - Chain = CurDAG->getCopyToReg(Chain, X86::AX, Move, InFlag); + Chain = CurDAG->getCopyToReg(Chain, X86::AX, Move, SDOperand()); InFlag = Chain.getValue(1); } else { AddToISelQueue(N0); InFlag = - CurDAG->getCopyToReg(CurDAG->getEntryNode(), LoReg, N0, - InFlag).getValue(1); + CurDAG->getCopyToReg(CurDAG->getEntryNode(), + LoReg, N0, SDOperand()).getValue(1); if (isSigned) { // Sign extend the low part into the high part. InFlag = @@ -1267,13 +1293,11 @@ SDNode *X86DAGToDAGISel::Select(SDOperand N) { } else { // Zero out the high part, effectively zero extending the input. SDOperand ClrNode = SDOperand(CurDAG->getTargetNode(ClrOpcode, NVT), 0); - InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), HiReg, ClrNode, - InFlag).getValue(1); + InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), HiReg, + ClrNode, InFlag).getValue(1); } } - SDOperand Tmp0, Tmp1, Tmp2, Tmp3, Chain; - bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3); if (foldedLoad) { AddToISelQueue(N1.getOperand(0)); AddToISelQueue(Tmp0); @@ -1283,19 +1307,19 @@ SDNode *X86DAGToDAGISel::Select(SDOperand N) { SDOperand Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, N1.getOperand(0), InFlag }; SDNode *CNode = CurDAG->getTargetNode(MOpc, MVT::Other, MVT::Flag, Ops, 6); - Chain = SDOperand(CNode, 0); InFlag = SDOperand(CNode, 1); + // Update the chain. + ReplaceUses(N1.getValue(1), SDOperand(CNode, 0)); } else { AddToISelQueue(N1); - Chain = CurDAG->getEntryNode(); InFlag = SDOperand(CurDAG->getTargetNode(Opc, MVT::Flag, N1, InFlag), 0); } // Copy the division (low) result, if it is needed. if (!N.getValue(0).use_empty()) { - SDOperand Result = CurDAG->getCopyFromReg(Chain, LoReg, NVT, InFlag); - Chain = Result.getValue(1); + SDOperand Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), + LoReg, NVT, InFlag); InFlag = Result.getValue(2); ReplaceUses(N.getValue(0), Result); #ifndef NDEBUG @@ -1310,8 +1334,8 @@ SDNode *X86DAGToDAGISel::Select(SDOperand N) { if (HiReg == X86::AH && Subtarget->is64Bit()) { // Prevent use of AH in a REX instruction by referencing AX instead. // Shift it down 8 bits. - Result = CurDAG->getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag); - Chain = Result.getValue(1); + Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), + X86::AX, MVT::i16, InFlag); InFlag = Result.getValue(2); Result = SDOperand(CurDAG->getTargetNode(X86::SHR16ri, MVT::i16, Result, CurDAG->getTargetConstant(8, MVT::i8)), 0); @@ -1320,8 +1344,8 @@ SDNode *X86DAGToDAGISel::Select(SDOperand N) { Result = SDOperand(CurDAG->getTargetNode(X86::EXTRACT_SUBREG, MVT::i8, Result, SRIdx), 0); } else { - Result = CurDAG->getCopyFromReg(Chain, HiReg, NVT, InFlag); - Chain = Result.getValue(1); + Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), + HiReg, NVT, InFlag); InFlag = Result.getValue(2); } ReplaceUses(N.getValue(1), Result); @@ -1331,8 +1355,6 @@ SDNode *X86DAGToDAGISel::Select(SDOperand N) { DOUT << "\n"; #endif } - if (foldedLoad) - ReplaceUses(N1.getValue(1), Chain); #ifndef NDEBUG Indent -= 2; diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index 63050731018..869e68bb208 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -154,26 +154,41 @@ X86TargetLowering::X86TargetLowering(TargetMachine &TM) setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand); } - // Divide and remainder are lowered to use div or idiv in legalize in - // order to expose the intermediate computations to trivial CSE. This is - // most noticeable when both x/y and x%y are being computed; they can be - // done with a single div or idiv. - setOperationAction(ISD::SDIV , MVT::i8 , Custom); - setOperationAction(ISD::UDIV , MVT::i8 , Custom); - setOperationAction(ISD::SREM , MVT::i8 , Custom); - setOperationAction(ISD::UREM , MVT::i8 , Custom); - setOperationAction(ISD::SDIV , MVT::i16 , Custom); - setOperationAction(ISD::UDIV , MVT::i16 , Custom); - setOperationAction(ISD::SREM , MVT::i16 , Custom); - setOperationAction(ISD::UREM , MVT::i16 , Custom); - setOperationAction(ISD::SDIV , MVT::i32 , Custom); - setOperationAction(ISD::UDIV , MVT::i32 , Custom); - setOperationAction(ISD::SREM , MVT::i32 , Custom); - setOperationAction(ISD::UREM , MVT::i32 , Custom); - setOperationAction(ISD::SDIV , MVT::i64 , Custom); - setOperationAction(ISD::UDIV , MVT::i64 , Custom); - setOperationAction(ISD::SREM , MVT::i64 , Custom); - setOperationAction(ISD::UREM , MVT::i64 , Custom); + // Scalar integer multiply, multiply-high, divide, and remainder are + // lowered to use operations that produce two results, to match the + // available instructions. This exposes the two-result form to trivial + // CSE, which is able to combine x/y and x%y into a single instruction, + // for example. The single-result multiply instructions are introduced + // in X86ISelDAGToDAG.cpp, after CSE, for uses where the the high part + // is not needed. + setOperationAction(ISD::MUL , MVT::i8 , Expand); + setOperationAction(ISD::MULHS , MVT::i8 , Expand); + setOperationAction(ISD::MULHU , MVT::i8 , Expand); + setOperationAction(ISD::SDIV , MVT::i8 , Expand); + setOperationAction(ISD::UDIV , MVT::i8 , Expand); + setOperationAction(ISD::SREM , MVT::i8 , Expand); + setOperationAction(ISD::UREM , MVT::i8 , Expand); + setOperationAction(ISD::MUL , MVT::i16 , Expand); + setOperationAction(ISD::MULHS , MVT::i16 , Expand); + setOperationAction(ISD::MULHU , MVT::i16 , Expand); + setOperationAction(ISD::SDIV , MVT::i16 , Expand); + setOperationAction(ISD::UDIV , MVT::i16 , Expand); + setOperationAction(ISD::SREM , MVT::i16 , Expand); + setOperationAction(ISD::UREM , MVT::i16 , Expand); + setOperationAction(ISD::MUL , MVT::i32 , Expand); + setOperationAction(ISD::MULHS , MVT::i32 , Expand); + setOperationAction(ISD::MULHU , MVT::i32 , Expand); + setOperationAction(ISD::SDIV , MVT::i32 , Expand); + setOperationAction(ISD::UDIV , MVT::i32 , Expand); + setOperationAction(ISD::SREM , MVT::i32 , Expand); + setOperationAction(ISD::UREM , MVT::i32 , Expand); + setOperationAction(ISD::MUL , MVT::i64 , Expand); + setOperationAction(ISD::MULHS , MVT::i64 , Expand); + setOperationAction(ISD::MULHU , MVT::i64 , Expand); + setOperationAction(ISD::SDIV , MVT::i64 , Expand); + setOperationAction(ISD::UDIV , MVT::i64 , Expand); + setOperationAction(ISD::SREM , MVT::i64 , Expand); + setOperationAction(ISD::UREM , MVT::i64 , Expand); setOperationAction(ISD::BR_JT , MVT::Other, Expand); setOperationAction(ISD::BRCOND , MVT::Other, Custom); @@ -449,6 +464,10 @@ X86TargetLowering::X86TargetLowering(TargetMachine &TM) setOperationAction(ISD::FPOWI, (MVT::ValueType)VT, Expand); setOperationAction(ISD::FSQRT, (MVT::ValueType)VT, Expand); setOperationAction(ISD::FCOPYSIGN, (MVT::ValueType)VT, Expand); + setOperationAction(ISD::SMUL_LOHI, (MVT::ValueType)VT, Expand); + setOperationAction(ISD::UMUL_LOHI, (MVT::ValueType)VT, Expand); + setOperationAction(ISD::SDIVREM, (MVT::ValueType)VT, Expand); + setOperationAction(ISD::UDIVREM, (MVT::ValueType)VT, Expand); } if (Subtarget->hasMMX()) { @@ -3398,22 +3417,6 @@ SDOperand X86TargetLowering::LowerShift(SDOperand Op, SelectionDAG &DAG) { return DAG.getNode(ISD::MERGE_VALUES, VTs, 2, &Ops[0], Ops.size()); } -SDOperand X86TargetLowering::LowerIntegerDivOrRem(SDOperand Op, SelectionDAG &DAG) { - unsigned Opcode = Op.getOpcode(); - MVT::ValueType NVT = Op.getValueType(); - bool isSigned = Opcode == ISD::SDIV || Opcode == ISD::SREM; - bool isDiv = Opcode == ISD::SDIV || Opcode == ISD::UDIV; - unsigned Opc = isSigned ? X86ISD::IDIV : X86ISD::DIV; - - SDOperand Ops[] = { Op.getOperand(0), Op.getOperand(1) }; - SDOperand DR = DAG.getNode(Opc, DAG.getVTList(NVT, NVT), Ops, 2); - - if (isDiv) - return DR; - - return SDOperand(DR.Val, 1); -} - SDOperand X86TargetLowering::LowerSINT_TO_FP(SDOperand Op, SelectionDAG &DAG) { assert(Op.getOperand(0).getValueType() <= MVT::i64 && Op.getOperand(0).getValueType() >= MVT::i16 && @@ -4545,10 +4548,6 @@ SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { case ISD::SHL_PARTS: case ISD::SRA_PARTS: case ISD::SRL_PARTS: return LowerShift(Op, DAG); - case ISD::SDIV: - case ISD::UDIV: - case ISD::SREM: - case ISD::UREM: return LowerIntegerDivOrRem(Op, DAG); case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); case ISD::FABS: return LowerFABS(Op, DAG); @@ -4620,8 +4619,6 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { case X86ISD::TLSADDR: return "X86ISD::TLSADDR"; case X86ISD::THREAD_POINTER: return "X86ISD::THREAD_POINTER"; case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN"; - case X86ISD::DIV: return "X86ISD::DIV"; - case X86ISD::IDIV: return "X86ISD::IDIV"; } } diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h index 9af46801d2c..2add24e297b 100644 --- a/lib/Target/X86/X86ISelLowering.h +++ b/lib/Target/X86/X86ISelLowering.h @@ -177,10 +177,6 @@ namespace llvm { /// in order to obtain suitable precision. FRSQRT, FRCP, - /// DIV, IDIV - Unsigned and signed integer division and remainder. - /// - DIV, IDIV, - // Thread Local Storage TLSADDR, THREAD_POINTER,