X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FTarget%2FARM%2FARMISelDAGToDAG.cpp;h=a458269b7c7ab5f50e39c9a3c2f38681b0190ee4;hb=518bb53485df640d7b7e3f6b0544099020c42aa7;hp=d236945f534b879fdf1676f1c7dc00d128428e92;hpb=764ab52dd80310a205c9888bf166d09dab858f90;p=oota-llvm.git diff --git a/lib/Target/ARM/ARMISelDAGToDAG.cpp b/lib/Target/ARM/ARMISelDAGToDAG.cpp index d236945f534..a458269b7c7 100644 --- a/lib/Target/ARM/ARMISelDAGToDAG.cpp +++ b/lib/Target/ARM/ARMISelDAGToDAG.cpp @@ -13,7 +13,6 @@ #include "ARM.h" #include "ARMAddressingModes.h" -#include "ARMConstantPoolValue.h" #include "ARMISelLowering.h" #include "ARMTargetMachine.h" #include "llvm/CallingConv.h" @@ -36,9 +35,6 @@ using namespace llvm; -static const unsigned arm_dsubreg_0 = 5; -static const unsigned arm_dsubreg_1 = 6; - //===--------------------------------------------------------------------===// /// ARMDAGToDAGISel - ARM specific code to select ARM machine /// instructions for SelectionDAG operations. @@ -52,8 +48,9 @@ class ARMDAGToDAGISel : public SelectionDAGISel { const ARMSubtarget *Subtarget; public: - explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm) - : SelectionDAGISel(tm), TM(tm), + explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, + CodeGenOpt::Level OptLevel) + : SelectionDAGISel(tm, OptLevel), TM(tm), Subtarget(&TM.getSubtarget()) { } @@ -61,58 +58,59 @@ public: return "ARM Instruction Selection"; } - /// getI32Imm - Return a target constant with the specified value, of type i32. + /// getI32Imm - Return a target constant of type i32 with the specified + /// value. inline SDValue getI32Imm(unsigned Imm) { - return CurDAG->getTargetConstant(Imm, EVT::i32); + return CurDAG->getTargetConstant(Imm, MVT::i32); } - SDNode *Select(SDValue Op); + SDNode *Select(SDNode *N); virtual void InstructionSelect(); - bool SelectShifterOperandReg(SDValue Op, SDValue N, SDValue &A, + bool SelectShifterOperandReg(SDNode *Op, SDValue N, SDValue &A, SDValue &B, SDValue &C); - bool SelectAddrMode2(SDValue Op, SDValue N, SDValue &Base, + bool SelectAddrMode2(SDNode *Op, SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc); - bool SelectAddrMode2Offset(SDValue Op, SDValue N, + bool SelectAddrMode2Offset(SDNode *Op, SDValue N, SDValue &Offset, SDValue &Opc); - bool SelectAddrMode3(SDValue Op, SDValue N, SDValue &Base, + bool SelectAddrMode3(SDNode *Op, SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc); - bool SelectAddrMode3Offset(SDValue Op, SDValue N, + bool SelectAddrMode3Offset(SDNode *Op, SDValue N, SDValue &Offset, SDValue &Opc); - bool SelectAddrMode4(SDValue Op, SDValue N, SDValue &Addr, + bool SelectAddrMode4(SDNode *Op, SDValue N, SDValue &Addr, SDValue &Mode); - bool SelectAddrMode5(SDValue Op, SDValue N, SDValue &Base, + bool SelectAddrMode5(SDNode *Op, SDValue N, SDValue &Base, SDValue &Offset); - bool SelectAddrMode6(SDValue Op, SDValue N, SDValue &Addr, SDValue &Update, - SDValue &Opc); + bool SelectAddrMode6(SDNode *Op, SDValue N, SDValue &Addr, SDValue &Update, + SDValue &Opc, SDValue &Align); - bool SelectAddrModePC(SDValue Op, SDValue N, SDValue &Offset, + bool SelectAddrModePC(SDNode *Op, SDValue N, SDValue &Offset, SDValue &Label); - bool SelectThumbAddrModeRR(SDValue Op, SDValue N, SDValue &Base, + bool SelectThumbAddrModeRR(SDNode *Op, SDValue N, SDValue &Base, SDValue &Offset); - bool SelectThumbAddrModeRI5(SDValue Op, SDValue N, unsigned Scale, + bool SelectThumbAddrModeRI5(SDNode *Op, SDValue N, unsigned Scale, SDValue &Base, SDValue &OffImm, SDValue &Offset); - bool SelectThumbAddrModeS1(SDValue Op, SDValue N, SDValue &Base, + bool SelectThumbAddrModeS1(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm, SDValue &Offset); - bool SelectThumbAddrModeS2(SDValue Op, SDValue N, SDValue &Base, + bool SelectThumbAddrModeS2(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm, SDValue &Offset); - bool SelectThumbAddrModeS4(SDValue Op, SDValue N, SDValue &Base, + bool SelectThumbAddrModeS4(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm, SDValue &Offset); - bool SelectThumbAddrModeSP(SDValue Op, SDValue N, SDValue &Base, + bool SelectThumbAddrModeSP(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm); - bool SelectT2ShifterOperandReg(SDValue Op, SDValue N, + bool SelectT2ShifterOperandReg(SDNode *Op, SDValue N, SDValue &BaseReg, SDValue &Opc); - bool SelectT2AddrModeImm12(SDValue Op, SDValue N, SDValue &Base, + bool SelectT2AddrModeImm12(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm); - bool SelectT2AddrModeImm8(SDValue Op, SDValue N, SDValue &Base, + bool SelectT2AddrModeImm8(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm); - bool SelectT2AddrModeImm8Offset(SDValue Op, SDValue N, + bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, SDValue &OffImm); - bool SelectT2AddrModeImm8s4(SDValue Op, SDValue N, SDValue &Base, + bool SelectT2AddrModeImm8s4(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm); - bool SelectT2AddrModeSoReg(SDValue Op, SDValue N, SDValue &Base, + bool SelectT2AddrModeSoReg(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffReg, SDValue &ShImm); // Include the pieces autogenerated from the target description. @@ -121,28 +119,94 @@ public: private: /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for /// ARM. - SDNode *SelectARMIndexedLoad(SDValue Op); - SDNode *SelectT2IndexedLoad(SDValue Op); + SDNode *SelectARMIndexedLoad(SDNode *N); + SDNode *SelectT2IndexedLoad(SDNode *N); /// SelectDYN_ALLOC - Select dynamic alloc for Thumb. - SDNode *SelectDYN_ALLOC(SDValue Op); + SDNode *SelectDYN_ALLOC(SDNode *N); + + /// SelectVLD - Select NEON load intrinsics. NumVecs should + /// be 2, 3 or 4. The opcode arrays specify the instructions used for + /// loads of D registers and even subregs and odd subregs of Q registers. + /// For NumVecs == 2, QOpcodes1 is not used. + SDNode *SelectVLD(SDNode *N, unsigned NumVecs, unsigned *DOpcodes, + unsigned *QOpcodes0, unsigned *QOpcodes1); + + /// SelectVST - Select NEON store intrinsics. NumVecs should + /// be 2, 3 or 4. The opcode arrays specify the instructions used for + /// stores of D registers and even subregs and odd subregs of Q registers. + /// For NumVecs == 2, QOpcodes1 is not used. + SDNode *SelectVST(SDNode *N, unsigned NumVecs, unsigned *DOpcodes, + unsigned *QOpcodes0, unsigned *QOpcodes1); + + /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should + /// be 2, 3 or 4. The opcode arrays specify the instructions used for + /// load/store of D registers and even subregs and odd subregs of Q registers. + SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, unsigned NumVecs, + unsigned *DOpcodes, unsigned *QOpcodes0, + unsigned *QOpcodes1); + + /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM. + SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, unsigned Opc); + + /// SelectCMOVOp - Select CMOV instructions for ARM. + SDNode *SelectCMOVOp(SDNode *N); + SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, + ARMCC::CondCodes CCVal, SDValue CCR, + SDValue InFlag); + SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, + ARMCC::CondCodes CCVal, SDValue CCR, + SDValue InFlag); + SDNode *SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, + ARMCC::CondCodes CCVal, SDValue CCR, + SDValue InFlag); + SDNode *SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, + ARMCC::CondCodes CCVal, SDValue CCR, + SDValue InFlag); /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for /// inline asm expressions. virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, std::vector &OutOps); + + /// PairDRegs - Insert a pair of double registers into an implicit def to + /// form a quad register. + SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1); }; } -void ARMDAGToDAGISel::InstructionSelect() { - DEBUG(BB->dump()); +/// isInt32Immediate - This method tests to see if the node is a 32-bit constant +/// operand. If so Imm will receive the 32-bit value. +static bool isInt32Immediate(SDNode *N, unsigned &Imm) { + if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { + Imm = cast(N)->getZExtValue(); + return true; + } + return false; +} +// isInt32Immediate - This method tests to see if a constant operand. +// If so Imm will receive the 32 bit value. +static bool isInt32Immediate(SDValue N, unsigned &Imm) { + return isInt32Immediate(N.getNode(), Imm); +} + +// isOpcWithIntImmediate - This method tests to see if the node is a specific +// opcode and that it has a immediate integer right operand. +// If so Imm will receive the 32 bit value. +static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { + return N->getOpcode() == Opc && + isInt32Immediate(N->getOperand(1).getNode(), Imm); +} + + +void ARMDAGToDAGISel::InstructionSelect() { SelectRoot(*CurDAG); CurDAG->RemoveDeadNodes(); } -bool ARMDAGToDAGISel::SelectShifterOperandReg(SDValue Op, +bool ARMDAGToDAGISel::SelectShifterOperandReg(SDNode *Op, SDValue N, SDValue &BaseReg, SDValue &ShReg, @@ -156,17 +220,17 @@ bool ARMDAGToDAGISel::SelectShifterOperandReg(SDValue Op, BaseReg = N.getOperand(0); unsigned ShImmVal = 0; if (ConstantSDNode *RHS = dyn_cast(N.getOperand(1))) { - ShReg = CurDAG->getRegister(0, EVT::i32); + ShReg = CurDAG->getRegister(0, MVT::i32); ShImmVal = RHS->getZExtValue() & 31; } else { ShReg = N.getOperand(1); } Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), - EVT::i32); + MVT::i32); return true; } -bool ARMDAGToDAGISel::SelectAddrMode2(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectAddrMode2(SDNode *Op, SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc) { if (N.getOpcode() == ISD::MUL) { @@ -185,7 +249,7 @@ bool ARMDAGToDAGISel::SelectAddrMode2(SDValue Op, SDValue N, Base = Offset = N.getOperand(0); Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ARM_AM::lsl), - EVT::i32); + MVT::i32); return true; } } @@ -197,13 +261,15 @@ bool ARMDAGToDAGISel::SelectAddrMode2(SDValue Op, SDValue N, if (N.getOpcode() == ISD::FrameIndex) { int FI = cast(N)->getIndex(); Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); - } else if (N.getOpcode() == ARMISD::Wrapper) { + } else if (N.getOpcode() == ARMISD::Wrapper && + !(Subtarget->useMovt() && + N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { Base = N.getOperand(0); } - Offset = CurDAG->getRegister(0, EVT::i32); + Offset = CurDAG->getRegister(0, MVT::i32); Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, ARM_AM::no_shift), - EVT::i32); + MVT::i32); return true; } @@ -218,7 +284,7 @@ bool ARMDAGToDAGISel::SelectAddrMode2(SDValue Op, SDValue N, int FI = cast(Base)->getIndex(); Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); } - Offset = CurDAG->getRegister(0, EVT::i32); + Offset = CurDAG->getRegister(0, MVT::i32); ARM_AM::AddrOpc AddSub = ARM_AM::add; if (RHSC < 0) { @@ -227,12 +293,12 @@ bool ARMDAGToDAGISel::SelectAddrMode2(SDValue Op, SDValue N, } Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC, ARM_AM::no_shift), - EVT::i32); + MVT::i32); return true; } } - // Otherwise this is R +/- [possibly shifted] R + // Otherwise this is R +/- [possibly shifted] R. ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub; ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1)); unsigned ShAmt = 0; @@ -270,13 +336,13 @@ bool ARMDAGToDAGISel::SelectAddrMode2(SDValue Op, SDValue N, } Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), - EVT::i32); + MVT::i32); return true; } -bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N, SDValue &Offset, SDValue &Opc) { - unsigned Opcode = Op.getOpcode(); + unsigned Opcode = Op->getOpcode(); ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) ? cast(Op)->getAddressingMode() : cast(Op)->getAddressingMode(); @@ -285,10 +351,10 @@ bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDValue Op, SDValue N, if (ConstantSDNode *C = dyn_cast(N)) { int Val = (int)C->getZExtValue(); if (Val >= 0 && Val < 0x1000) { // 12 bits. - Offset = CurDAG->getRegister(0, EVT::i32); + Offset = CurDAG->getRegister(0, MVT::i32); Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val, ARM_AM::no_shift), - EVT::i32); + MVT::i32); return true; } } @@ -308,19 +374,19 @@ bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDValue Op, SDValue N, } Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), - EVT::i32); + MVT::i32); return true; } -bool ARMDAGToDAGISel::SelectAddrMode3(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectAddrMode3(SDNode *Op, SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc) { if (N.getOpcode() == ISD::SUB) { // X - C is canonicalize to X + -C, no need to handle it here. Base = N.getOperand(0); Offset = N.getOperand(1); - Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),EVT::i32); + Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32); return true; } @@ -330,8 +396,8 @@ bool ARMDAGToDAGISel::SelectAddrMode3(SDValue Op, SDValue N, int FI = cast(N)->getIndex(); Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); } - Offset = CurDAG->getRegister(0, EVT::i32); - Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),EVT::i32); + Offset = CurDAG->getRegister(0, MVT::i32); + Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32); return true; } @@ -345,27 +411,27 @@ bool ARMDAGToDAGISel::SelectAddrMode3(SDValue Op, SDValue N, int FI = cast(Base)->getIndex(); Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); } - Offset = CurDAG->getRegister(0, EVT::i32); + Offset = CurDAG->getRegister(0, MVT::i32); ARM_AM::AddrOpc AddSub = ARM_AM::add; if (RHSC < 0) { AddSub = ARM_AM::sub; RHSC = - RHSC; } - Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),EVT::i32); + Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32); return true; } } Base = N.getOperand(0); Offset = N.getOperand(1); - Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), EVT::i32); + Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32); return true; } -bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N, SDValue &Offset, SDValue &Opc) { - unsigned Opcode = Op.getOpcode(); + unsigned Opcode = Op->getOpcode(); ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) ? cast(Op)->getAddressingMode() : cast(Op)->getAddressingMode(); @@ -374,36 +440,38 @@ bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDValue Op, SDValue N, if (ConstantSDNode *C = dyn_cast(N)) { int Val = (int)C->getZExtValue(); if (Val >= 0 && Val < 256) { - Offset = CurDAG->getRegister(0, EVT::i32); - Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), EVT::i32); + Offset = CurDAG->getRegister(0, MVT::i32); + Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32); return true; } } Offset = N; - Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), EVT::i32); + Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32); return true; } -bool ARMDAGToDAGISel::SelectAddrMode4(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectAddrMode4(SDNode *Op, SDValue N, SDValue &Addr, SDValue &Mode) { Addr = N; - Mode = CurDAG->getTargetConstant(0, EVT::i32); + Mode = CurDAG->getTargetConstant(0, MVT::i32); return true; } -bool ARMDAGToDAGISel::SelectAddrMode5(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectAddrMode5(SDNode *Op, SDValue N, SDValue &Base, SDValue &Offset) { if (N.getOpcode() != ISD::ADD) { Base = N; if (N.getOpcode() == ISD::FrameIndex) { int FI = cast(N)->getIndex(); Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); - } else if (N.getOpcode() == ARMISD::Wrapper) { + } else if (N.getOpcode() == ARMISD::Wrapper && + !(Subtarget->useMovt() && + N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { Base = N.getOperand(0); } Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), - EVT::i32); + MVT::i32); return true; } @@ -426,7 +494,7 @@ bool ARMDAGToDAGISel::SelectAddrMode5(SDValue Op, SDValue N, RHSC = - RHSC; } Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC), - EVT::i32); + MVT::i32); return true; } } @@ -434,36 +502,38 @@ bool ARMDAGToDAGISel::SelectAddrMode5(SDValue Op, SDValue N, Base = N; Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), - EVT::i32); + MVT::i32); return true; } -bool ARMDAGToDAGISel::SelectAddrMode6(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Op, SDValue N, SDValue &Addr, SDValue &Update, - SDValue &Opc) { + SDValue &Opc, SDValue &Align) { Addr = N; - // The optional writeback is handled in ARMLoadStoreOpt. - Update = CurDAG->getRegister(0, EVT::i32); - Opc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(false), EVT::i32); + // Default to no writeback. + Update = CurDAG->getRegister(0, MVT::i32); + Opc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(false), MVT::i32); + // Default to no alignment. + Align = CurDAG->getTargetConstant(0, MVT::i32); return true; } -bool ARMDAGToDAGISel::SelectAddrModePC(SDValue Op, SDValue N, - SDValue &Offset, SDValue &Label) { +bool ARMDAGToDAGISel::SelectAddrModePC(SDNode *Op, SDValue N, + SDValue &Offset, SDValue &Label) { if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) { Offset = N.getOperand(0); SDValue N1 = N.getOperand(1); Label = CurDAG->getTargetConstant(cast(N1)->getZExtValue(), - EVT::i32); + MVT::i32); return true; } return false; } -bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDNode *Op, SDValue N, SDValue &Base, SDValue &Offset){ // FIXME dl should come from the parent load or store, not the address - DebugLoc dl = Op.getDebugLoc(); + DebugLoc dl = Op->getDebugLoc(); if (N.getOpcode() != ISD::ADD) { ConstantSDNode *NC = dyn_cast(N); if (!NC || NC->getZExtValue() != 0) @@ -479,7 +549,7 @@ bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue Op, SDValue N, } bool -ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDValue Op, SDValue N, +ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDNode *Op, SDValue N, unsigned Scale, SDValue &Base, SDValue &OffImm, SDValue &Offset) { if (Scale == 4) { @@ -492,9 +562,15 @@ ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDValue Op, SDValue N, } if (N.getOpcode() != ISD::ADD) { - Base = (N.getOpcode() == ARMISD::Wrapper) ? N.getOperand(0) : N; - Offset = CurDAG->getRegister(0, EVT::i32); - OffImm = CurDAG->getTargetConstant(0, EVT::i32); + if (N.getOpcode() == ARMISD::Wrapper && + !(Subtarget->useMovt() && + N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { + Base = N.getOperand(0); + } else + Base = N; + + Offset = CurDAG->getRegister(0, MVT::i32); + OffImm = CurDAG->getTargetConstant(0, MVT::i32); return true; } @@ -504,8 +580,8 @@ ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDValue Op, SDValue N, if ((LHSR && LHSR->getReg() == ARM::SP) || (RHSR && RHSR->getReg() == ARM::SP)) { Base = N; - Offset = CurDAG->getRegister(0, EVT::i32); - OffImm = CurDAG->getTargetConstant(0, EVT::i32); + Offset = CurDAG->getRegister(0, MVT::i32); + OffImm = CurDAG->getTargetConstant(0, MVT::i32); return true; } @@ -516,8 +592,8 @@ ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDValue Op, SDValue N, RHSC /= Scale; if (RHSC >= 0 && RHSC < 32) { Base = N.getOperand(0); - Offset = CurDAG->getRegister(0, EVT::i32); - OffImm = CurDAG->getTargetConstant(RHSC, EVT::i32); + Offset = CurDAG->getRegister(0, MVT::i32); + OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); return true; } } @@ -525,34 +601,34 @@ ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDValue Op, SDValue N, Base = N.getOperand(0); Offset = N.getOperand(1); - OffImm = CurDAG->getTargetConstant(0, EVT::i32); + OffImm = CurDAG->getTargetConstant(0, MVT::i32); return true; } -bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm, SDValue &Offset) { return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset); } -bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm, SDValue &Offset) { return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset); } -bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm, SDValue &Offset) { return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset); } -bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm) { if (N.getOpcode() == ISD::FrameIndex) { int FI = cast(N)->getIndex(); Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); - OffImm = CurDAG->getTargetConstant(0, EVT::i32); + OffImm = CurDAG->getTargetConstant(0, MVT::i32); return true; } @@ -573,7 +649,7 @@ bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue Op, SDValue N, int FI = cast(Base)->getIndex(); Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); } - OffImm = CurDAG->getTargetConstant(RHSC, EVT::i32); + OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); return true; } } @@ -583,7 +659,7 @@ bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue Op, SDValue N, return false; } -bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDNode *Op, SDValue N, SDValue &BaseReg, SDValue &Opc) { ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N); @@ -603,7 +679,7 @@ bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue Op, SDValue N, return false; } -bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm) { // Match simple R + imm12 operands. @@ -613,15 +689,17 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue Op, SDValue N, // Match frame index... int FI = cast(N)->getIndex(); Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); - OffImm = CurDAG->getTargetConstant(0, EVT::i32); + OffImm = CurDAG->getTargetConstant(0, MVT::i32); return true; - } else if (N.getOpcode() == ARMISD::Wrapper) { + } else if (N.getOpcode() == ARMISD::Wrapper && + !(Subtarget->useMovt() && + N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { Base = N.getOperand(0); if (Base.getOpcode() == ISD::TargetConstantPool) return false; // We want to select t2LDRpci instead. } else Base = N; - OffImm = CurDAG->getTargetConstant(0, EVT::i32); + OffImm = CurDAG->getTargetConstant(0, MVT::i32); return true; } @@ -640,18 +718,18 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue Op, SDValue N, int FI = cast(Base)->getIndex(); Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); } - OffImm = CurDAG->getTargetConstant(RHSC, EVT::i32); + OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); return true; } } // Base only. Base = N; - OffImm = CurDAG->getTargetConstant(0, EVT::i32); + OffImm = CurDAG->getTargetConstant(0, MVT::i32); return true; } -bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm) { // Match simple R - imm8 operands. if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) { @@ -666,7 +744,7 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue Op, SDValue N, int FI = cast(Base)->getIndex(); Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); } - OffImm = CurDAG->getTargetConstant(RHSC, EVT::i32); + OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); return true; } } @@ -675,9 +753,9 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue Op, SDValue N, return false; } -bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, SDValue &OffImm){ - unsigned Opcode = Op.getOpcode(); + unsigned Opcode = Op->getOpcode(); ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) ? cast(Op)->getAddressingMode() : cast(Op)->getAddressingMode(); @@ -685,8 +763,8 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDValue Op, SDValue N, int RHSC = (int)RHS->getZExtValue(); if (RHSC >= 0 && RHSC < 0x100) { // 8 bits. OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC)) - ? CurDAG->getTargetConstant(RHSC, EVT::i32) - : CurDAG->getTargetConstant(-RHSC, EVT::i32); + ? CurDAG->getTargetConstant(RHSC, MVT::i32) + : CurDAG->getTargetConstant(-RHSC, MVT::i32); return true; } } @@ -694,7 +772,7 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDValue Op, SDValue N, return false; } -bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffImm) { if (N.getOpcode() == ISD::ADD) { if (ConstantSDNode *RHS = dyn_cast(N.getOperand(1))) { @@ -702,7 +780,7 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDValue Op, SDValue N, if (((RHSC & 0x3) == 0) && ((RHSC >= 0 && RHSC < 0x400) || (RHSC < 0 && RHSC > -0x400))) { // 8 bits. Base = N.getOperand(0); - OffImm = CurDAG->getTargetConstant(RHSC, EVT::i32); + OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); return true; } } @@ -711,7 +789,7 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDValue Op, SDValue N, int RHSC = (int)RHS->getZExtValue(); if (((RHSC & 0x3) == 0) && (RHSC >= 0 && RHSC < 0x400)) { // 8 bits. Base = N.getOperand(0); - OffImm = CurDAG->getTargetConstant(-RHSC, EVT::i32); + OffImm = CurDAG->getTargetConstant(-RHSC, MVT::i32); return true; } } @@ -720,7 +798,7 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDValue Op, SDValue N, return false; } -bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue Op, SDValue N, +bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDNode *Op, SDValue N, SDValue &Base, SDValue &OffReg, SDValue &ShImm) { // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12. @@ -764,7 +842,7 @@ bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue Op, SDValue N, } } - ShImm = CurDAG->getTargetConstant(ShAmt, EVT::i32); + ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32); return true; } @@ -773,11 +851,11 @@ bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue Op, SDValue N, /// getAL - Returns a ARMCC::AL immediate node. static inline SDValue getAL(SelectionDAG *CurDAG) { - return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, EVT::i32); + return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32); } -SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDValue Op) { - LoadSDNode *LD = cast(Op); +SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) { + LoadSDNode *LD = cast(N); ISD::MemIndexedMode AM = LD->getAddressingMode(); if (AM == ISD::UNINDEXED) return NULL; @@ -787,24 +865,24 @@ SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDValue Op) { bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); unsigned Opcode = 0; bool Match = false; - if (LoadedVT == EVT::i32 && - SelectAddrMode2Offset(Op, LD->getOffset(), Offset, AMOpc)) { + if (LoadedVT == MVT::i32 && + SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) { Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST; Match = true; - } else if (LoadedVT == EVT::i16 && - SelectAddrMode3Offset(Op, LD->getOffset(), Offset, AMOpc)) { + } else if (LoadedVT == MVT::i16 && + SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { Match = true; Opcode = (LD->getExtensionType() == ISD::SEXTLOAD) ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST) : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST); - } else if (LoadedVT == EVT::i8 || LoadedVT == EVT::i1) { + } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) { if (LD->getExtensionType() == ISD::SEXTLOAD) { - if (SelectAddrMode3Offset(Op, LD->getOffset(), Offset, AMOpc)) { + if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { Match = true; Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST; } } else { - if (SelectAddrMode2Offset(Op, LD->getOffset(), Offset, AMOpc)) { + if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) { Match = true; Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST; } @@ -815,16 +893,16 @@ SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDValue Op) { SDValue Chain = LD->getChain(); SDValue Base = LD->getBasePtr(); SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG), - CurDAG->getRegister(0, EVT::i32), Chain }; - return CurDAG->getTargetNode(Opcode, Op.getDebugLoc(), EVT::i32, EVT::i32, - EVT::Other, Ops, 6); + CurDAG->getRegister(0, MVT::i32), Chain }; + return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32, + MVT::Other, Ops, 6); } return NULL; } -SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDValue Op) { - LoadSDNode *LD = cast(Op); +SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) { + LoadSDNode *LD = cast(N); ISD::MemIndexedMode AM = LD->getAddressingMode(); if (AM == ISD::UNINDEXED) return NULL; @@ -835,19 +913,19 @@ SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDValue Op) { bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); unsigned Opcode = 0; bool Match = false; - if (SelectT2AddrModeImm8Offset(Op, LD->getOffset(), Offset)) { - switch (LoadedVT.getSimpleVT()) { - case EVT::i32: + if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) { + switch (LoadedVT.getSimpleVT().SimpleTy) { + case MVT::i32: Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST; break; - case EVT::i16: + case MVT::i16: if (isSExtLd) Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST; else Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST; break; - case EVT::i8: - case EVT::i1: + case MVT::i8: + case MVT::i1: if (isSExtLd) Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST; else @@ -863,29 +941,28 @@ SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDValue Op) { SDValue Chain = LD->getChain(); SDValue Base = LD->getBasePtr(); SDValue Ops[]= { Base, Offset, getAL(CurDAG), - CurDAG->getRegister(0, EVT::i32), Chain }; - return CurDAG->getTargetNode(Opcode, Op.getDebugLoc(), EVT::i32, EVT::i32, - EVT::Other, Ops, 5); + CurDAG->getRegister(0, MVT::i32), Chain }; + return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32, + MVT::Other, Ops, 5); } return NULL; } -SDNode *ARMDAGToDAGISel::SelectDYN_ALLOC(SDValue Op) { - SDNode *N = Op.getNode(); +SDNode *ARMDAGToDAGISel::SelectDYN_ALLOC(SDNode *N) { DebugLoc dl = N->getDebugLoc(); - EVT VT = Op.getValueType(); - SDValue Chain = Op.getOperand(0); - SDValue Size = Op.getOperand(1); - SDValue Align = Op.getOperand(2); - SDValue SP = CurDAG->getRegister(ARM::SP, EVT::i32); + EVT VT = N->getValueType(0); + SDValue Chain = N->getOperand(0); + SDValue Size = N->getOperand(1); + SDValue Align = N->getOperand(2); + SDValue SP = CurDAG->getRegister(ARM::SP, MVT::i32); int32_t AlignVal = cast(Align)->getSExtValue(); if (AlignVal < 0) // We need to align the stack. Use Thumb1 tAND which is the only thumb // instruction that can read and write SP. This matches to a pseudo // instruction that has a chain to ensure the result is written back to // the stack pointer. - SP = SDValue(CurDAG->getTargetNode(ARM::tANDsp, dl, VT, SP, Align), 0); + SP = SDValue(CurDAG->getMachineNode(ARM::tANDsp, dl, VT, SP, Align), 0); bool isC = isa(Size); uint32_t C = isC ? cast(Size)->getZExtValue() : ~0UL; @@ -893,31 +970,31 @@ SDNode *ARMDAGToDAGISel::SelectDYN_ALLOC(SDValue Op) { // tSUBspi - immediate is between 0 ... 508 inclusive. if (C <= 508 && ((C & 3) == 0)) // FIXME: tSUBspi encode scale 4 implicitly. - return CurDAG->SelectNodeTo(N, ARM::tSUBspi_, VT, EVT::Other, SP, - CurDAG->getTargetConstant(C/4, EVT::i32), + return CurDAG->SelectNodeTo(N, ARM::tSUBspi_, VT, MVT::Other, SP, + CurDAG->getTargetConstant(C/4, MVT::i32), Chain); if (Subtarget->isThumb1Only()) { - // Use tADDrSPr since Thumb1 does not have a sub r, sp, r. ARMISelLowering + // Use tADDspr since Thumb1 does not have a sub r, sp, r. ARMISelLowering // should have negated the size operand already. FIXME: We can't insert // new target independent node at this stage so we are forced to negate // it earlier. Is there a better solution? - return CurDAG->SelectNodeTo(N, ARM::tADDspr_, VT, EVT::Other, SP, Size, + return CurDAG->SelectNodeTo(N, ARM::tADDspr_, VT, MVT::Other, SP, Size, Chain); } else if (Subtarget->isThumb2()) { if (isC && Predicate_t2_so_imm(Size.getNode())) { // t2SUBrSPi - SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, EVT::i32), Chain }; - return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi_, VT, EVT::Other, Ops, 3); + SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain }; + return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi_, VT, MVT::Other, Ops, 3); } else if (isC && Predicate_imm0_4095(Size.getNode())) { // t2SUBrSPi12 - SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, EVT::i32), Chain }; - return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi12_, VT, EVT::Other, Ops, 3); + SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain }; + return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi12_, VT, MVT::Other, Ops, 3); } else { // t2SUBrSPs SDValue Ops[] = { SP, Size, getI32Imm(ARM_AM::getSORegOpc(ARM_AM::lsl,0)), Chain }; - return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPs_, VT, EVT::Other, Ops, 4); + return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPs_, VT, MVT::Other, Ops, 4); } } @@ -925,8 +1002,518 @@ SDNode *ARMDAGToDAGISel::SelectDYN_ALLOC(SDValue Op) { return 0; } -SDNode *ARMDAGToDAGISel::Select(SDValue Op) { - SDNode *N = Op.getNode(); +/// PairDRegs - Insert a pair of double registers into an implicit def to +/// form a quad register. +SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) { + DebugLoc dl = V0.getNode()->getDebugLoc(); + SDValue Undef = + SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0); + SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32); + SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32); + SDNode *Pair = CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl, + VT, Undef, V0, SubReg0); + return CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl, + VT, SDValue(Pair, 0), V1, SubReg1); +} + +/// GetNEONSubregVT - Given a type for a 128-bit NEON vector, return the type +/// for a 64-bit subregister of the vector. +static EVT GetNEONSubregVT(EVT VT) { + switch (VT.getSimpleVT().SimpleTy) { + default: llvm_unreachable("unhandled NEON type"); + case MVT::v16i8: return MVT::v8i8; + case MVT::v8i16: return MVT::v4i16; + case MVT::v4f32: return MVT::v2f32; + case MVT::v4i32: return MVT::v2i32; + case MVT::v2i64: return MVT::v1i64; + } +} + +SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs, + unsigned *DOpcodes, unsigned *QOpcodes0, + unsigned *QOpcodes1) { + assert(NumVecs >=2 && NumVecs <= 4 && "VLD NumVecs out-of-range"); + DebugLoc dl = N->getDebugLoc(); + + SDValue MemAddr, MemUpdate, MemOpc, Align; + if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, MemUpdate, MemOpc, Align)) + return NULL; + + SDValue Chain = N->getOperand(0); + EVT VT = N->getValueType(0); + bool is64BitVector = VT.is64BitVector(); + + unsigned OpcodeIndex; + switch (VT.getSimpleVT().SimpleTy) { + default: llvm_unreachable("unhandled vld type"); + // Double-register operations: + case MVT::v8i8: OpcodeIndex = 0; break; + case MVT::v4i16: OpcodeIndex = 1; break; + case MVT::v2f32: + case MVT::v2i32: OpcodeIndex = 2; break; + case MVT::v1i64: OpcodeIndex = 3; break; + // Quad-register operations: + case MVT::v16i8: OpcodeIndex = 0; break; + case MVT::v8i16: OpcodeIndex = 1; break; + case MVT::v4f32: + case MVT::v4i32: OpcodeIndex = 2; break; + } + + SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32); + SDValue PredReg = CurDAG->getRegister(0, MVT::i32); + if (is64BitVector) { + unsigned Opc = DOpcodes[OpcodeIndex]; + const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Align, + Pred, PredReg, Chain }; + std::vector ResTys(NumVecs, VT); + ResTys.push_back(MVT::Other); + return CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 7); + } + + EVT RegVT = GetNEONSubregVT(VT); + if (NumVecs == 2) { + // Quad registers are directly supported for VLD2, + // loading 2 pairs of D regs. + unsigned Opc = QOpcodes0[OpcodeIndex]; + const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Align, + Pred, PredReg, Chain }; + std::vector ResTys(4, VT); + ResTys.push_back(MVT::Other); + SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 7); + Chain = SDValue(VLd, 4); + + // Combine the even and odd subregs to produce the result. + for (unsigned Vec = 0; Vec < NumVecs; ++Vec) { + SDNode *Q = PairDRegs(VT, SDValue(VLd, 2*Vec), SDValue(VLd, 2*Vec+1)); + ReplaceUses(SDValue(N, Vec), SDValue(Q, 0)); + } + } else { + // Otherwise, quad registers are loaded with two separate instructions, + // where one loads the even registers and the other loads the odd registers. + + // Enable writeback to the address register. + MemOpc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(true), MVT::i32); + + std::vector ResTys(NumVecs, RegVT); + ResTys.push_back(MemAddr.getValueType()); + ResTys.push_back(MVT::Other); + + // Load the even subregs. + unsigned Opc = QOpcodes0[OpcodeIndex]; + const SDValue OpsA[] = { MemAddr, MemUpdate, MemOpc, Align, + Pred, PredReg, Chain }; + SDNode *VLdA = CurDAG->getMachineNode(Opc, dl, ResTys, OpsA, 7); + Chain = SDValue(VLdA, NumVecs+1); + + // Load the odd subregs. + Opc = QOpcodes1[OpcodeIndex]; + const SDValue OpsB[] = { SDValue(VLdA, NumVecs), MemUpdate, MemOpc, + Align, Pred, PredReg, Chain }; + SDNode *VLdB = CurDAG->getMachineNode(Opc, dl, ResTys, OpsB, 7); + Chain = SDValue(VLdB, NumVecs+1); + + // Combine the even and odd subregs to produce the result. + for (unsigned Vec = 0; Vec < NumVecs; ++Vec) { + SDNode *Q = PairDRegs(VT, SDValue(VLdA, Vec), SDValue(VLdB, Vec)); + ReplaceUses(SDValue(N, Vec), SDValue(Q, 0)); + } + } + ReplaceUses(SDValue(N, NumVecs), Chain); + return NULL; +} + +SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs, + unsigned *DOpcodes, unsigned *QOpcodes0, + unsigned *QOpcodes1) { + assert(NumVecs >=2 && NumVecs <= 4 && "VST NumVecs out-of-range"); + DebugLoc dl = N->getDebugLoc(); + + SDValue MemAddr, MemUpdate, MemOpc, Align; + if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, MemUpdate, MemOpc, Align)) + return NULL; + + SDValue Chain = N->getOperand(0); + EVT VT = N->getOperand(3).getValueType(); + bool is64BitVector = VT.is64BitVector(); + + unsigned OpcodeIndex; + switch (VT.getSimpleVT().SimpleTy) { + default: llvm_unreachable("unhandled vst type"); + // Double-register operations: + case MVT::v8i8: OpcodeIndex = 0; break; + case MVT::v4i16: OpcodeIndex = 1; break; + case MVT::v2f32: + case MVT::v2i32: OpcodeIndex = 2; break; + case MVT::v1i64: OpcodeIndex = 3; break; + // Quad-register operations: + case MVT::v16i8: OpcodeIndex = 0; break; + case MVT::v8i16: OpcodeIndex = 1; break; + case MVT::v4f32: + case MVT::v4i32: OpcodeIndex = 2; break; + } + + SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32); + SDValue PredReg = CurDAG->getRegister(0, MVT::i32); + + SmallVector Ops; + Ops.push_back(MemAddr); + Ops.push_back(MemUpdate); + Ops.push_back(MemOpc); + Ops.push_back(Align); + + if (is64BitVector) { + unsigned Opc = DOpcodes[OpcodeIndex]; + for (unsigned Vec = 0; Vec < NumVecs; ++Vec) + Ops.push_back(N->getOperand(Vec+3)); + Ops.push_back(Pred); + Ops.push_back(PredReg); + Ops.push_back(Chain); + return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+7); + } + + EVT RegVT = GetNEONSubregVT(VT); + if (NumVecs == 2) { + // Quad registers are directly supported for VST2, + // storing 2 pairs of D regs. + unsigned Opc = QOpcodes0[OpcodeIndex]; + for (unsigned Vec = 0; Vec < NumVecs; ++Vec) { + Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT, + N->getOperand(Vec+3))); + Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT, + N->getOperand(Vec+3))); + } + Ops.push_back(Pred); + Ops.push_back(PredReg); + Ops.push_back(Chain); + return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 11); + } + + // Otherwise, quad registers are stored with two separate instructions, + // where one stores the even registers and the other stores the odd registers. + + // Enable writeback to the address register. + MemOpc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(true), MVT::i32); + + // Store the even subregs. + for (unsigned Vec = 0; Vec < NumVecs; ++Vec) + Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT, + N->getOperand(Vec+3))); + Ops.push_back(Pred); + Ops.push_back(PredReg); + Ops.push_back(Chain); + unsigned Opc = QOpcodes0[OpcodeIndex]; + SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(), + MVT::Other, Ops.data(), NumVecs+7); + Chain = SDValue(VStA, 1); + + // Store the odd subregs. + Ops[0] = SDValue(VStA, 0); // MemAddr + for (unsigned Vec = 0; Vec < NumVecs; ++Vec) + Ops[Vec+4] = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT, + N->getOperand(Vec+3)); + Ops[NumVecs+4] = Pred; + Ops[NumVecs+5] = PredReg; + Ops[NumVecs+6] = Chain; + Opc = QOpcodes1[OpcodeIndex]; + SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(), + MVT::Other, Ops.data(), NumVecs+7); + Chain = SDValue(VStB, 1); + ReplaceUses(SDValue(N, 0), Chain); + return NULL; +} + +SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, + unsigned NumVecs, unsigned *DOpcodes, + unsigned *QOpcodes0, + unsigned *QOpcodes1) { + assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range"); + DebugLoc dl = N->getDebugLoc(); + + SDValue MemAddr, MemUpdate, MemOpc, Align; + if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, MemUpdate, MemOpc, Align)) + return NULL; + + SDValue Chain = N->getOperand(0); + unsigned Lane = + cast(N->getOperand(NumVecs+3))->getZExtValue(); + EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType(); + bool is64BitVector = VT.is64BitVector(); + + // Quad registers are handled by load/store of subregs. Find the subreg info. + unsigned NumElts = 0; + int SubregIdx = 0; + EVT RegVT = VT; + if (!is64BitVector) { + RegVT = GetNEONSubregVT(VT); + NumElts = RegVT.getVectorNumElements(); + SubregIdx = (Lane < NumElts) ? ARM::DSUBREG_0 : ARM::DSUBREG_1; + } + + unsigned OpcodeIndex; + switch (VT.getSimpleVT().SimpleTy) { + default: llvm_unreachable("unhandled vld/vst lane type"); + // Double-register operations: + case MVT::v8i8: OpcodeIndex = 0; break; + case MVT::v4i16: OpcodeIndex = 1; break; + case MVT::v2f32: + case MVT::v2i32: OpcodeIndex = 2; break; + // Quad-register operations: + case MVT::v8i16: OpcodeIndex = 0; break; + case MVT::v4f32: + case MVT::v4i32: OpcodeIndex = 1; break; + } + + SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32); + SDValue PredReg = CurDAG->getRegister(0, MVT::i32); + + SmallVector Ops; + Ops.push_back(MemAddr); + Ops.push_back(MemUpdate); + Ops.push_back(MemOpc); + Ops.push_back(Align); + + unsigned Opc = 0; + if (is64BitVector) { + Opc = DOpcodes[OpcodeIndex]; + for (unsigned Vec = 0; Vec < NumVecs; ++Vec) + Ops.push_back(N->getOperand(Vec+3)); + } else { + // Check if this is loading the even or odd subreg of a Q register. + if (Lane < NumElts) { + Opc = QOpcodes0[OpcodeIndex]; + } else { + Lane -= NumElts; + Opc = QOpcodes1[OpcodeIndex]; + } + // Extract the subregs of the input vector. + for (unsigned Vec = 0; Vec < NumVecs; ++Vec) + Ops.push_back(CurDAG->getTargetExtractSubreg(SubregIdx, dl, RegVT, + N->getOperand(Vec+3))); + } + Ops.push_back(getI32Imm(Lane)); + Ops.push_back(Pred); + Ops.push_back(PredReg); + Ops.push_back(Chain); + + if (!IsLoad) + return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+8); + + std::vector ResTys(NumVecs, RegVT); + ResTys.push_back(MVT::Other); + SDNode *VLdLn = + CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), NumVecs+8); + // For a 64-bit vector load to D registers, nothing more needs to be done. + if (is64BitVector) + return VLdLn; + + // For 128-bit vectors, take the 64-bit results of the load and insert them + // as subregs into the result. + for (unsigned Vec = 0; Vec < NumVecs; ++Vec) { + SDValue QuadVec = CurDAG->getTargetInsertSubreg(SubregIdx, dl, VT, + N->getOperand(Vec+3), + SDValue(VLdLn, Vec)); + ReplaceUses(SDValue(N, Vec), QuadVec); + } + + Chain = SDValue(VLdLn, NumVecs); + ReplaceUses(SDValue(N, NumVecs), Chain); + return NULL; +} + +SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N, + unsigned Opc) { + if (!Subtarget->hasV6T2Ops()) + return NULL; + + unsigned Shl_imm = 0; + if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) { + assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!"); + unsigned Srl_imm = 0; + if (isInt32Immediate(N->getOperand(1), Srl_imm)) { + assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); + unsigned Width = 32 - Srl_imm; + int LSB = Srl_imm - Shl_imm; + if (LSB < 0) + return NULL; + SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); + SDValue Ops[] = { N->getOperand(0).getOperand(0), + CurDAG->getTargetConstant(LSB, MVT::i32), + CurDAG->getTargetConstant(Width, MVT::i32), + getAL(CurDAG), Reg0 }; + return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); + } + } + return NULL; +} + +SDNode *ARMDAGToDAGISel:: +SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, + ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { + SDValue CPTmp0; + SDValue CPTmp1; + if (SelectT2ShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1)) { + unsigned SOVal = cast(CPTmp1)->getZExtValue(); + unsigned SOShOp = ARM_AM::getSORegShOp(SOVal); + unsigned Opc = 0; + switch (SOShOp) { + case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break; + case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break; + case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break; + case ARM_AM::ror: Opc = ARM::t2MOVCCror; break; + default: + llvm_unreachable("Unknown so_reg opcode!"); + break; + } + SDValue SOShImm = + CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32); + SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); + SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag }; + return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6); + } + return 0; +} + +SDNode *ARMDAGToDAGISel:: +SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, + ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { + SDValue CPTmp0; + SDValue CPTmp1; + SDValue CPTmp2; + if (SelectShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1, CPTmp2)) { + SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); + SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag }; + return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7); + } + return 0; +} + +SDNode *ARMDAGToDAGISel:: +SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, + ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { + ConstantSDNode *T = dyn_cast(TrueVal); + if (!T) + return 0; + + if (Predicate_t2_so_imm(TrueVal.getNode())) { + SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32); + SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); + SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag }; + return CurDAG->SelectNodeTo(N, + ARM::t2MOVCCi, MVT::i32, Ops, 5); + } + return 0; +} + +SDNode *ARMDAGToDAGISel:: +SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, + ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { + ConstantSDNode *T = dyn_cast(TrueVal); + if (!T) + return 0; + + if (Predicate_so_imm(TrueVal.getNode())) { + SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32); + SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); + SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag }; + return CurDAG->SelectNodeTo(N, + ARM::MOVCCi, MVT::i32, Ops, 5); + } + return 0; +} + +SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) { + EVT VT = N->getValueType(0); + SDValue FalseVal = N->getOperand(0); + SDValue TrueVal = N->getOperand(1); + SDValue CC = N->getOperand(2); + SDValue CCR = N->getOperand(3); + SDValue InFlag = N->getOperand(4); + assert(CC.getOpcode() == ISD::Constant); + assert(CCR.getOpcode() == ISD::Register); + ARMCC::CondCodes CCVal = + (ARMCC::CondCodes)cast(CC)->getZExtValue(); + + if (!Subtarget->isThumb1Only() && VT == MVT::i32) { + // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc) + // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc) + // Pattern complexity = 18 cost = 1 size = 0 + SDValue CPTmp0; + SDValue CPTmp1; + SDValue CPTmp2; + if (Subtarget->isThumb()) { + SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal, + CCVal, CCR, InFlag); + if (!Res) + Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal, + ARMCC::getOppositeCondition(CCVal), CCR, InFlag); + if (Res) + return Res; + } else { + SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal, + CCVal, CCR, InFlag); + if (!Res) + Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal, + ARMCC::getOppositeCondition(CCVal), CCR, InFlag); + if (Res) + return Res; + } + + // Pattern: (ARMcmov:i32 GPR:i32:$false, + // (imm:i32)<>:$true, + // (imm:i32):$cc) + // Emits: (MOVCCi:i32 GPR:i32:$false, + // (so_imm:i32 (imm:i32):$true), (imm:i32):$cc) + // Pattern complexity = 10 cost = 1 size = 0 + if (Subtarget->isThumb()) { + SDNode *Res = SelectT2CMOVSoImmOp(N, FalseVal, TrueVal, + CCVal, CCR, InFlag); + if (!Res) + Res = SelectT2CMOVSoImmOp(N, TrueVal, FalseVal, + ARMCC::getOppositeCondition(CCVal), CCR, InFlag); + if (Res) + return Res; + } else { + SDNode *Res = SelectARMCMOVSoImmOp(N, FalseVal, TrueVal, + CCVal, CCR, InFlag); + if (!Res) + Res = SelectARMCMOVSoImmOp(N, TrueVal, FalseVal, + ARMCC::getOppositeCondition(CCVal), CCR, InFlag); + if (Res) + return Res; + } + } + + // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) + // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) + // Pattern complexity = 6 cost = 1 size = 0 + // + // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) + // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) + // Pattern complexity = 6 cost = 11 size = 0 + // + // Also FCPYScc and FCPYDcc. + SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32); + SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag }; + unsigned Opc = 0; + switch (VT.getSimpleVT().SimpleTy) { + default: assert(false && "Illegal conditional move type!"); + break; + case MVT::i32: + Opc = Subtarget->isThumb() + ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo) + : ARM::MOVCCr; + break; + case MVT::f32: + Opc = ARM::VMOVScc; + break; + case MVT::f64: + Opc = ARM::VMOVDcc; + break; + } + return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5); +} + +SDNode *ARMDAGToDAGISel::Select(SDNode *N) { DebugLoc dl = N->getDebugLoc(); if (N->isMachineOpcode()) @@ -937,44 +1524,47 @@ SDNode *ARMDAGToDAGISel::Select(SDValue Op) { case ISD::Constant: { unsigned Val = cast(N)->getZExtValue(); bool UseCP = true; - if (Subtarget->isThumb()) { - if (Subtarget->hasThumb2()) - // Thumb2 has the MOVT instruction, so all immediates can - // be done with MOV + MOVT, at worst. - UseCP = 0; - else + if (Subtarget->hasThumb2()) + // Thumb2-aware targets have the MOVT instruction, so all immediates can + // be done with MOV + MOVT, at worst. + UseCP = 0; + else { + if (Subtarget->isThumb()) { UseCP = (Val > 255 && // MOV ~Val > 255 && // MOV + MVN !ARM_AM::isThumbImmShiftedVal(Val)); // MOV + LSL - } else - UseCP = (ARM_AM::getSOImmVal(Val) == -1 && // MOV - ARM_AM::getSOImmVal(~Val) == -1 && // MVN - !ARM_AM::isSOImmTwoPartVal(Val)); // two instrs. + } else + UseCP = (ARM_AM::getSOImmVal(Val) == -1 && // MOV + ARM_AM::getSOImmVal(~Val) == -1 && // MVN + !ARM_AM::isSOImmTwoPartVal(Val)); // two instrs. + } + if (UseCP) { SDValue CPIdx = - CurDAG->getTargetConstantPool(ConstantInt::get(Type::Int32Ty, Val), + CurDAG->getTargetConstantPool(ConstantInt::get( + Type::getInt32Ty(*CurDAG->getContext()), Val), TLI.getPointerTy()); SDNode *ResNode; if (Subtarget->isThumb1Only()) { - SDValue Pred = CurDAG->getTargetConstant(0xEULL, EVT::i32); - SDValue PredReg = CurDAG->getRegister(0, EVT::i32); + SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32); + SDValue PredReg = CurDAG->getRegister(0, MVT::i32); SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() }; - ResNode = CurDAG->getTargetNode(ARM::tLDRcp, dl, EVT::i32, EVT::Other, - Ops, 4); + ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other, + Ops, 4); } else { SDValue Ops[] = { CPIdx, - CurDAG->getRegister(0, EVT::i32), - CurDAG->getTargetConstant(0, EVT::i32), + CurDAG->getRegister(0, MVT::i32), + CurDAG->getTargetConstant(0, MVT::i32), getAL(CurDAG), - CurDAG->getRegister(0, EVT::i32), + CurDAG->getRegister(0, MVT::i32), CurDAG->getEntryNode() }; - ResNode=CurDAG->getTargetNode(ARM::LDRcp, dl, EVT::i32, EVT::Other, - Ops, 6); + ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other, + Ops, 6); } - ReplaceUses(Op, SDValue(ResNode, 0)); + ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0)); return NULL; } @@ -986,98 +1576,145 @@ SDNode *ARMDAGToDAGISel::Select(SDValue Op) { int FI = cast(N)->getIndex(); SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); if (Subtarget->isThumb1Only()) { - return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, EVT::i32, TFI, - CurDAG->getTargetConstant(0, EVT::i32)); + return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI, + CurDAG->getTargetConstant(0, MVT::i32)); } else { unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ? ARM::t2ADDri : ARM::ADDri); - SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, EVT::i32), - getAL(CurDAG), CurDAG->getRegister(0, EVT::i32), - CurDAG->getRegister(0, EVT::i32) }; - return CurDAG->SelectNodeTo(N, Opc, EVT::i32, Ops, 5); + SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32), + getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), + CurDAG->getRegister(0, MVT::i32) }; + return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); } } case ARMISD::DYN_ALLOC: - return SelectDYN_ALLOC(Op); + return SelectDYN_ALLOC(N); + case ISD::SRL: + if (SDNode *I = SelectV6T2BitfieldExtractOp(N, + Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX)) + return I; + break; + case ISD::SRA: + if (SDNode *I = SelectV6T2BitfieldExtractOp(N, + Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)) + return I; + break; case ISD::MUL: if (Subtarget->isThumb1Only()) break; - if (ConstantSDNode *C = dyn_cast(Op.getOperand(1))) { + if (ConstantSDNode *C = dyn_cast(N->getOperand(1))) { unsigned RHSV = C->getZExtValue(); if (!RHSV) break; if (isPowerOf2_32(RHSV-1)) { // 2^n+1? unsigned ShImm = Log2_32(RHSV-1); if (ShImm >= 32) break; - SDValue V = Op.getOperand(0); + SDValue V = N->getOperand(0); ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); - SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, EVT::i32); - SDValue Reg0 = CurDAG->getRegister(0, EVT::i32); + SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); + SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); if (Subtarget->isThumb()) { SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; - return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, EVT::i32, Ops, 6); + return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6); } else { SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; - return CurDAG->SelectNodeTo(N, ARM::ADDrs, EVT::i32, Ops, 7); + return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7); } } if (isPowerOf2_32(RHSV+1)) { // 2^n-1? unsigned ShImm = Log2_32(RHSV+1); if (ShImm >= 32) break; - SDValue V = Op.getOperand(0); + SDValue V = N->getOperand(0); ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); - SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, EVT::i32); - SDValue Reg0 = CurDAG->getRegister(0, EVT::i32); + SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); + SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); if (Subtarget->isThumb()) { SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0 }; - return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, EVT::i32, Ops, 5); + return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 5); } else { SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; - return CurDAG->SelectNodeTo(N, ARM::RSBrs, EVT::i32, Ops, 7); + return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7); } } } break; - case ARMISD::FMRRD: - return CurDAG->getTargetNode(ARM::FMRRD, dl, EVT::i32, EVT::i32, - Op.getOperand(0), getAL(CurDAG), - CurDAG->getRegister(0, EVT::i32)); + case ISD::AND: { + // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits + // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits + // are entirely contributed by c2 and lower 16-bits are entirely contributed + // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)). + // Select it to: "movt x, ((c1 & 0xffff) >> 16) + EVT VT = N->getValueType(0); + if (VT != MVT::i32) + break; + unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2()) + ? ARM::t2MOVTi16 + : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0); + if (!Opc) + break; + SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); + ConstantSDNode *N1C = dyn_cast(N1); + if (!N1C) + break; + if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) { + SDValue N2 = N0.getOperand(1); + ConstantSDNode *N2C = dyn_cast(N2); + if (!N2C) + break; + unsigned N1CVal = N1C->getZExtValue(); + unsigned N2CVal = N2C->getZExtValue(); + if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) && + (N1CVal & 0xffffU) == 0xffffU && + (N2CVal & 0xffffU) == 0x0U) { + SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16, + MVT::i32); + SDValue Ops[] = { N0.getOperand(0), Imm16, + getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; + return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4); + } + } + break; + } + case ARMISD::VMOVRRD: + return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32, + N->getOperand(0), getAL(CurDAG), + CurDAG->getRegister(0, MVT::i32)); case ISD::UMUL_LOHI: { if (Subtarget->isThumb1Only()) break; if (Subtarget->isThumb()) { - SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1), - getAL(CurDAG), CurDAG->getRegister(0, EVT::i32), - CurDAG->getRegister(0, EVT::i32) }; - return CurDAG->getTargetNode(ARM::t2UMULL, dl, EVT::i32, EVT::i32, Ops,4); + SDValue Ops[] = { N->getOperand(0), N->getOperand(1), + getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), + CurDAG->getRegister(0, MVT::i32) }; + return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops,4); } else { - SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1), - getAL(CurDAG), CurDAG->getRegister(0, EVT::i32), - CurDAG->getRegister(0, EVT::i32) }; - return CurDAG->getTargetNode(ARM::UMULL, dl, EVT::i32, EVT::i32, Ops, 5); + SDValue Ops[] = { N->getOperand(0), N->getOperand(1), + getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), + CurDAG->getRegister(0, MVT::i32) }; + return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5); } } case ISD::SMUL_LOHI: { if (Subtarget->isThumb1Only()) break; if (Subtarget->isThumb()) { - SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1), - getAL(CurDAG), CurDAG->getRegister(0, EVT::i32) }; - return CurDAG->getTargetNode(ARM::t2SMULL, dl, EVT::i32, EVT::i32, Ops,4); + SDValue Ops[] = { N->getOperand(0), N->getOperand(1), + getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; + return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops,4); } else { - SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1), - getAL(CurDAG), CurDAG->getRegister(0, EVT::i32), - CurDAG->getRegister(0, EVT::i32) }; - return CurDAG->getTargetNode(ARM::SMULL, dl, EVT::i32, EVT::i32, Ops, 5); + SDValue Ops[] = { N->getOperand(0), N->getOperand(1), + getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), + CurDAG->getRegister(0, MVT::i32) }; + return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5); } } case ISD::LOAD: { SDNode *ResNode = 0; if (Subtarget->isThumb() && Subtarget->hasThumb2()) - ResNode = SelectT2IndexedLoad(Op); + ResNode = SelectT2IndexedLoad(N); else - ResNode = SelectARMIndexedLoad(Op); + ResNode = SelectARMIndexedLoad(N); if (ResNode) return ResNode; // Other cases are autogenerated. @@ -1098,437 +1735,234 @@ SDNode *ARMDAGToDAGISel::Select(SDValue Op) { unsigned Opc = Subtarget->isThumb() ? ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc; - SDValue Chain = Op.getOperand(0); - SDValue N1 = Op.getOperand(1); - SDValue N2 = Op.getOperand(2); - SDValue N3 = Op.getOperand(3); - SDValue InFlag = Op.getOperand(4); + SDValue Chain = N->getOperand(0); + SDValue N1 = N->getOperand(1); + SDValue N2 = N->getOperand(2); + SDValue N3 = N->getOperand(3); + SDValue InFlag = N->getOperand(4); assert(N1.getOpcode() == ISD::BasicBlock); assert(N2.getOpcode() == ISD::Constant); assert(N3.getOpcode() == ISD::Register); SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) cast(N2)->getZExtValue()), - EVT::i32); + MVT::i32); SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag }; - SDNode *ResNode = CurDAG->getTargetNode(Opc, dl, EVT::Other, - EVT::Flag, Ops, 5); + SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, + MVT::Flag, Ops, 5); Chain = SDValue(ResNode, 0); - if (Op.getNode()->getNumValues() == 2) { + if (N->getNumValues() == 2) { InFlag = SDValue(ResNode, 1); - ReplaceUses(SDValue(Op.getNode(), 1), InFlag); + ReplaceUses(SDValue(N, 1), InFlag); } - ReplaceUses(SDValue(Op.getNode(), 0), SDValue(Chain.getNode(), Chain.getResNo())); + ReplaceUses(SDValue(N, 0), + SDValue(Chain.getNode(), Chain.getResNo())); return NULL; } - case ARMISD::CMOV: { - EVT VT = Op.getValueType(); - SDValue N0 = Op.getOperand(0); - SDValue N1 = Op.getOperand(1); - SDValue N2 = Op.getOperand(2); - SDValue N3 = Op.getOperand(3); - SDValue InFlag = Op.getOperand(4); + case ARMISD::CMOV: + return SelectCMOVOp(N); + case ARMISD::CNEG: { + EVT VT = N->getValueType(0); + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + SDValue N2 = N->getOperand(2); + SDValue N3 = N->getOperand(3); + SDValue InFlag = N->getOperand(4); assert(N2.getOpcode() == ISD::Constant); assert(N3.getOpcode() == ISD::Register); - if (!Subtarget->isThumb1Only() && VT == EVT::i32) { - // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc) - // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc) - // Pattern complexity = 18 cost = 1 size = 0 - SDValue CPTmp0; - SDValue CPTmp1; - SDValue CPTmp2; - if (Subtarget->isThumb()) { - if (SelectT2ShifterOperandReg(Op, N1, CPTmp0, CPTmp1)) { - unsigned SOVal = cast(CPTmp1)->getZExtValue(); - unsigned SOShOp = ARM_AM::getSORegShOp(SOVal); - unsigned Opc = 0; - switch (SOShOp) { - case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break; - case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break; - case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break; - case ARM_AM::ror: Opc = ARM::t2MOVCCror; break; - default: - llvm_unreachable("Unknown so_reg opcode!"); - break; - } - SDValue SOShImm = - CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), EVT::i32); - SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) - cast(N2)->getZExtValue()), - EVT::i32); - SDValue Ops[] = { N0, CPTmp0, SOShImm, Tmp2, N3, InFlag }; - return CurDAG->SelectNodeTo(Op.getNode(), Opc, EVT::i32,Ops, 6); - } - } else { - if (SelectShifterOperandReg(Op, N1, CPTmp0, CPTmp1, CPTmp2)) { - SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) - cast(N2)->getZExtValue()), - EVT::i32); - SDValue Ops[] = { N0, CPTmp0, CPTmp1, CPTmp2, Tmp2, N3, InFlag }; - return CurDAG->SelectNodeTo(Op.getNode(), - ARM::MOVCCs, EVT::i32, Ops, 7); - } - } - - // Pattern: (ARMcmov:i32 GPR:i32:$false, - // (imm:i32)<>:$true, - // (imm:i32):$cc) - // Emits: (MOVCCi:i32 GPR:i32:$false, - // (so_imm:i32 (imm:i32):$true), (imm:i32):$cc) - // Pattern complexity = 10 cost = 1 size = 0 - if (N3.getOpcode() == ISD::Constant) { - if (Subtarget->isThumb()) { - if (Predicate_t2_so_imm(N3.getNode())) { - SDValue Tmp1 = CurDAG->getTargetConstant(((unsigned) - cast(N1)->getZExtValue()), - EVT::i32); - SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) - cast(N2)->getZExtValue()), - EVT::i32); - SDValue Ops[] = { N0, Tmp1, Tmp2, N3, InFlag }; - return CurDAG->SelectNodeTo(Op.getNode(), - ARM::t2MOVCCi, EVT::i32, Ops, 5); - } - } else { - if (Predicate_so_imm(N3.getNode())) { - SDValue Tmp1 = CurDAG->getTargetConstant(((unsigned) - cast(N1)->getZExtValue()), - EVT::i32); - SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) - cast(N2)->getZExtValue()), - EVT::i32); - SDValue Ops[] = { N0, Tmp1, Tmp2, N3, InFlag }; - return CurDAG->SelectNodeTo(Op.getNode(), - ARM::MOVCCi, EVT::i32, Ops, 5); - } - } - } - } - - // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) - // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) - // Pattern complexity = 6 cost = 1 size = 0 - // - // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) - // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) - // Pattern complexity = 6 cost = 11 size = 0 - // - // Also FCPYScc and FCPYDcc. SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) cast(N2)->getZExtValue()), - EVT::i32); + MVT::i32); SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag }; unsigned Opc = 0; - switch (VT.getSimpleVT()) { + switch (VT.getSimpleVT().SimpleTy) { default: assert(false && "Illegal conditional move type!"); break; - case EVT::i32: - Opc = Subtarget->isThumb() - ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr) - : ARM::MOVCCr; - break; - case EVT::f32: - Opc = ARM::FCPYScc; + case MVT::f32: + Opc = ARM::VNEGScc; break; - case EVT::f64: - Opc = ARM::FCPYDcc; + case MVT::f64: + Opc = ARM::VNEGDcc; break; } - return CurDAG->SelectNodeTo(Op.getNode(), Opc, VT, Ops, 5); + return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5); } - case ARMISD::CNEG: { - EVT VT = Op.getValueType(); - SDValue N0 = Op.getOperand(0); - SDValue N1 = Op.getOperand(1); - SDValue N2 = Op.getOperand(2); - SDValue N3 = Op.getOperand(3); - SDValue InFlag = Op.getOperand(4); - assert(N2.getOpcode() == ISD::Constant); - assert(N3.getOpcode() == ISD::Register); - SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) - cast(N2)->getZExtValue()), - EVT::i32); - SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag }; + case ARMISD::VZIP: { unsigned Opc = 0; - switch (VT.getSimpleVT()) { - default: assert(false && "Illegal conditional move type!"); - break; - case EVT::f32: - Opc = ARM::FNEGScc; - break; - case EVT::f64: - Opc = ARM::FNEGDcc; - break; + EVT VT = N->getValueType(0); + switch (VT.getSimpleVT().SimpleTy) { + default: return NULL; + case MVT::v8i8: Opc = ARM::VZIPd8; break; + case MVT::v4i16: Opc = ARM::VZIPd16; break; + case MVT::v2f32: + case MVT::v2i32: Opc = ARM::VZIPd32; break; + case MVT::v16i8: Opc = ARM::VZIPq8; break; + case MVT::v8i16: Opc = ARM::VZIPq16; break; + case MVT::v4f32: + case MVT::v4i32: Opc = ARM::VZIPq32; break; } - return CurDAG->SelectNodeTo(Op.getNode(), Opc, VT, Ops, 5); + SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32); + SDValue PredReg = CurDAG->getRegister(0, MVT::i32); + SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; + return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4); } - - case ISD::DECLARE: { - SDValue Chain = Op.getOperand(0); - SDValue N1 = Op.getOperand(1); - SDValue N2 = Op.getOperand(2); - FrameIndexSDNode *FINode = dyn_cast(N1); - // FIXME: handle VLAs. - if (!FINode) { - ReplaceUses(Op.getValue(0), Chain); - return NULL; + case ARMISD::VUZP: { + unsigned Opc = 0; + EVT VT = N->getValueType(0); + switch (VT.getSimpleVT().SimpleTy) { + default: return NULL; + case MVT::v8i8: Opc = ARM::VUZPd8; break; + case MVT::v4i16: Opc = ARM::VUZPd16; break; + case MVT::v2f32: + case MVT::v2i32: Opc = ARM::VUZPd32; break; + case MVT::v16i8: Opc = ARM::VUZPq8; break; + case MVT::v8i16: Opc = ARM::VUZPq16; break; + case MVT::v4f32: + case MVT::v4i32: Opc = ARM::VUZPq32; break; } - if (N2.getOpcode() == ARMISD::PIC_ADD && isa(N2.getOperand(0))) - N2 = N2.getOperand(0); - LoadSDNode *Ld = dyn_cast(N2); - if (!Ld) { - ReplaceUses(Op.getValue(0), Chain); - return NULL; + SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32); + SDValue PredReg = CurDAG->getRegister(0, MVT::i32); + SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; + return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4); + } + case ARMISD::VTRN: { + unsigned Opc = 0; + EVT VT = N->getValueType(0); + switch (VT.getSimpleVT().SimpleTy) { + default: return NULL; + case MVT::v8i8: Opc = ARM::VTRNd8; break; + case MVT::v4i16: Opc = ARM::VTRNd16; break; + case MVT::v2f32: + case MVT::v2i32: Opc = ARM::VTRNd32; break; + case MVT::v16i8: Opc = ARM::VTRNq8; break; + case MVT::v8i16: Opc = ARM::VTRNq16; break; + case MVT::v4f32: + case MVT::v4i32: Opc = ARM::VTRNq32; break; } - SDValue BasePtr = Ld->getBasePtr(); - assert(BasePtr.getOpcode() == ARMISD::Wrapper && - isa(BasePtr.getOperand(0)) && - "llvm.dbg.variable should be a constantpool node"); - ConstantPoolSDNode *CP = cast(BasePtr.getOperand(0)); - GlobalValue *GV = 0; - if (CP->isMachineConstantPoolEntry()) { - ARMConstantPoolValue *ACPV = (ARMConstantPoolValue*)CP->getMachineCPVal(); - GV = ACPV->getGV(); - } else - GV = dyn_cast(CP->getConstVal()); - if (!GV) { - ReplaceUses(Op.getValue(0), Chain); - return NULL; + SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32); + SDValue PredReg = CurDAG->getRegister(0, MVT::i32); + SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; + return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4); + } + + case ISD::INTRINSIC_VOID: + case ISD::INTRINSIC_W_CHAIN: { + unsigned IntNo = cast(N->getOperand(1))->getZExtValue(); + switch (IntNo) { + default: + break; + + case Intrinsic::arm_neon_vld2: { + unsigned DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16, + ARM::VLD2d32, ARM::VLD2d64 }; + unsigned QOpcodes[] = { ARM::VLD2q8, ARM::VLD2q16, ARM::VLD2q32 }; + return SelectVLD(N, 2, DOpcodes, QOpcodes, 0); } - SDValue Tmp1 = CurDAG->getTargetFrameIndex(FINode->getIndex(), - TLI.getPointerTy()); - SDValue Tmp2 = CurDAG->getTargetGlobalAddress(GV, TLI.getPointerTy()); - SDValue Ops[] = { Tmp1, Tmp2, Chain }; - return CurDAG->getTargetNode(TargetInstrInfo::DECLARE, dl, - EVT::Other, Ops, 3); - } - - case ISD::VECTOR_SHUFFLE: { - EVT VT = Op.getValueType(); - - // Match 128-bit splat to VDUPLANEQ. (This could be done with a Pat in - // ARMInstrNEON.td but it is awkward because the shuffle mask needs to be - // transformed first into a lane number and then to both a subregister - // index and an adjusted lane number.) If the source operand is a - // SCALAR_TO_VECTOR, leave it so it will be matched later as a VDUP. - ShuffleVectorSDNode *SVOp = cast(N); - if (VT.is128BitVector() && SVOp->isSplat() && - Op.getOperand(0).getOpcode() != ISD::SCALAR_TO_VECTOR && - Op.getOperand(1).getOpcode() == ISD::UNDEF) { - unsigned LaneVal = SVOp->getSplatIndex(); - - EVT HalfVT; - unsigned Opc = 0; - switch (VT.getVectorElementType().getSimpleVT()) { - default: llvm_unreachable("unhandled VDUP splat type"); - case EVT::i8: Opc = ARM::VDUPLN8q; HalfVT = EVT::v8i8; break; - case EVT::i16: Opc = ARM::VDUPLN16q; HalfVT = EVT::v4i16; break; - case EVT::i32: Opc = ARM::VDUPLN32q; HalfVT = EVT::v2i32; break; - case EVT::f32: Opc = ARM::VDUPLNfq; HalfVT = EVT::v2f32; break; - } + case Intrinsic::arm_neon_vld3: { + unsigned DOpcodes[] = { ARM::VLD3d8, ARM::VLD3d16, + ARM::VLD3d32, ARM::VLD3d64 }; + unsigned QOpcodes0[] = { ARM::VLD3q8a, ARM::VLD3q16a, ARM::VLD3q32a }; + unsigned QOpcodes1[] = { ARM::VLD3q8b, ARM::VLD3q16b, ARM::VLD3q32b }; + return SelectVLD(N, 3, DOpcodes, QOpcodes0, QOpcodes1); + } - // The source operand needs to be changed to a subreg of the original - // 128-bit operand, and the lane number needs to be adjusted accordingly. - unsigned NumElts = VT.getVectorNumElements() / 2; - unsigned SRVal = (LaneVal < NumElts ? arm_dsubreg_0 : arm_dsubreg_1); - SDValue SR = CurDAG->getTargetConstant(SRVal, EVT::i32); - SDValue NewLane = CurDAG->getTargetConstant(LaneVal % NumElts, EVT::i32); - SDNode *SubReg = CurDAG->getTargetNode(TargetInstrInfo::EXTRACT_SUBREG, - dl, HalfVT, N->getOperand(0), SR); - return CurDAG->SelectNodeTo(N, Opc, VT, SDValue(SubReg, 0), NewLane); + case Intrinsic::arm_neon_vld4: { + unsigned DOpcodes[] = { ARM::VLD4d8, ARM::VLD4d16, + ARM::VLD4d32, ARM::VLD4d64 }; + unsigned QOpcodes0[] = { ARM::VLD4q8a, ARM::VLD4q16a, ARM::VLD4q32a }; + unsigned QOpcodes1[] = { ARM::VLD4q8b, ARM::VLD4q16b, ARM::VLD4q32b }; + return SelectVLD(N, 4, DOpcodes, QOpcodes0, QOpcodes1); } - break; - } + case Intrinsic::arm_neon_vld2lane: { + unsigned DOpcodes[] = { ARM::VLD2LNd8, ARM::VLD2LNd16, ARM::VLD2LNd32 }; + unsigned QOpcodes0[] = { ARM::VLD2LNq16a, ARM::VLD2LNq32a }; + unsigned QOpcodes1[] = { ARM::VLD2LNq16b, ARM::VLD2LNq32b }; + return SelectVLDSTLane(N, true, 2, DOpcodes, QOpcodes0, QOpcodes1); + } - case ARMISD::VLD2D: { - SDValue MemAddr, MemUpdate, MemOpc; - if (!SelectAddrMode6(Op, N->getOperand(1), MemAddr, MemUpdate, MemOpc)) - return NULL; - unsigned Opc = 0; - EVT VT = Op.getValueType(); - switch (VT.getSimpleVT()) { - default: llvm_unreachable("unhandled VLD2D type"); - case EVT::v8i8: Opc = ARM::VLD2d8; break; - case EVT::v4i16: Opc = ARM::VLD2d16; break; - case EVT::v2f32: - case EVT::v2i32: Opc = ARM::VLD2d32; break; + case Intrinsic::arm_neon_vld3lane: { + unsigned DOpcodes[] = { ARM::VLD3LNd8, ARM::VLD3LNd16, ARM::VLD3LNd32 }; + unsigned QOpcodes0[] = { ARM::VLD3LNq16a, ARM::VLD3LNq32a }; + unsigned QOpcodes1[] = { ARM::VLD3LNq16b, ARM::VLD3LNq32b }; + return SelectVLDSTLane(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); } - const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc }; - return CurDAG->getTargetNode(Opc, dl, VT, VT, EVT::Other, Ops, 3); - } - case ARMISD::VLD3D: { - SDValue MemAddr, MemUpdate, MemOpc; - if (!SelectAddrMode6(Op, N->getOperand(1), MemAddr, MemUpdate, MemOpc)) - return NULL; - unsigned Opc = 0; - EVT VT = Op.getValueType(); - switch (VT.getSimpleVT()) { - default: llvm_unreachable("unhandled VLD3D type"); - case EVT::v8i8: Opc = ARM::VLD3d8; break; - case EVT::v4i16: Opc = ARM::VLD3d16; break; - case EVT::v2f32: - case EVT::v2i32: Opc = ARM::VLD3d32; break; + case Intrinsic::arm_neon_vld4lane: { + unsigned DOpcodes[] = { ARM::VLD4LNd8, ARM::VLD4LNd16, ARM::VLD4LNd32 }; + unsigned QOpcodes0[] = { ARM::VLD4LNq16a, ARM::VLD4LNq32a }; + unsigned QOpcodes1[] = { ARM::VLD4LNq16b, ARM::VLD4LNq32b }; + return SelectVLDSTLane(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); } - const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc }; - return CurDAG->getTargetNode(Opc, dl, VT, VT, VT, EVT::Other, Ops, 3); - } - case ARMISD::VLD4D: { - SDValue MemAddr, MemUpdate, MemOpc; - if (!SelectAddrMode6(Op, N->getOperand(1), MemAddr, MemUpdate, MemOpc)) - return NULL; - unsigned Opc = 0; - EVT VT = Op.getValueType(); - switch (VT.getSimpleVT()) { - default: llvm_unreachable("unhandled VLD4D type"); - case EVT::v8i8: Opc = ARM::VLD4d8; break; - case EVT::v4i16: Opc = ARM::VLD4d16; break; - case EVT::v2f32: - case EVT::v2i32: Opc = ARM::VLD4d32; break; + case Intrinsic::arm_neon_vst2: { + unsigned DOpcodes[] = { ARM::VST2d8, ARM::VST2d16, + ARM::VST2d32, ARM::VST2d64 }; + unsigned QOpcodes[] = { ARM::VST2q8, ARM::VST2q16, ARM::VST2q32 }; + return SelectVST(N, 2, DOpcodes, QOpcodes, 0); } - const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc }; - std::vector ResTys(4, VT); - ResTys.push_back(EVT::Other); - return CurDAG->getTargetNode(Opc, dl, ResTys, Ops, 3); - } - case ARMISD::VST2D: { - SDValue MemAddr, MemUpdate, MemOpc; - if (!SelectAddrMode6(Op, N->getOperand(1), MemAddr, MemUpdate, MemOpc)) - return NULL; - unsigned Opc = 0; - switch (N->getOperand(2).getValueType().getSimpleVT()) { - default: llvm_unreachable("unhandled VST2D type"); - case EVT::v8i8: Opc = ARM::VST2d8; break; - case EVT::v4i16: Opc = ARM::VST2d16; break; - case EVT::v2f32: - case EVT::v2i32: Opc = ARM::VST2d32; break; + case Intrinsic::arm_neon_vst3: { + unsigned DOpcodes[] = { ARM::VST3d8, ARM::VST3d16, + ARM::VST3d32, ARM::VST3d64 }; + unsigned QOpcodes0[] = { ARM::VST3q8a, ARM::VST3q16a, ARM::VST3q32a }; + unsigned QOpcodes1[] = { ARM::VST3q8b, ARM::VST3q16b, ARM::VST3q32b }; + return SelectVST(N, 3, DOpcodes, QOpcodes0, QOpcodes1); } - const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, - N->getOperand(2), N->getOperand(3) }; - return CurDAG->getTargetNode(Opc, dl, EVT::Other, Ops, 5); - } - case ARMISD::VST3D: { - SDValue MemAddr, MemUpdate, MemOpc; - if (!SelectAddrMode6(Op, N->getOperand(1), MemAddr, MemUpdate, MemOpc)) - return NULL; - unsigned Opc = 0; - switch (N->getOperand(2).getValueType().getSimpleVT()) { - default: llvm_unreachable("unhandled VST3D type"); - case EVT::v8i8: Opc = ARM::VST3d8; break; - case EVT::v4i16: Opc = ARM::VST3d16; break; - case EVT::v2f32: - case EVT::v2i32: Opc = ARM::VST3d32; break; + case Intrinsic::arm_neon_vst4: { + unsigned DOpcodes[] = { ARM::VST4d8, ARM::VST4d16, + ARM::VST4d32, ARM::VST4d64 }; + unsigned QOpcodes0[] = { ARM::VST4q8a, ARM::VST4q16a, ARM::VST4q32a }; + unsigned QOpcodes1[] = { ARM::VST4q8b, ARM::VST4q16b, ARM::VST4q32b }; + return SelectVST(N, 4, DOpcodes, QOpcodes0, QOpcodes1); } - const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, - N->getOperand(2), N->getOperand(3), - N->getOperand(4) }; - return CurDAG->getTargetNode(Opc, dl, EVT::Other, Ops, 6); - } - case ARMISD::VST4D: { - SDValue MemAddr, MemUpdate, MemOpc; - if (!SelectAddrMode6(Op, N->getOperand(1), MemAddr, MemUpdate, MemOpc)) - return NULL; - unsigned Opc = 0; - switch (N->getOperand(2).getValueType().getSimpleVT()) { - default: llvm_unreachable("unhandled VST4D type"); - case EVT::v8i8: Opc = ARM::VST4d8; break; - case EVT::v4i16: Opc = ARM::VST4d16; break; - case EVT::v2f32: - case EVT::v2i32: Opc = ARM::VST4d32; break; + case Intrinsic::arm_neon_vst2lane: { + unsigned DOpcodes[] = { ARM::VST2LNd8, ARM::VST2LNd16, ARM::VST2LNd32 }; + unsigned QOpcodes0[] = { ARM::VST2LNq16a, ARM::VST2LNq32a }; + unsigned QOpcodes1[] = { ARM::VST2LNq16b, ARM::VST2LNq32b }; + return SelectVLDSTLane(N, false, 2, DOpcodes, QOpcodes0, QOpcodes1); } - const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, - N->getOperand(2), N->getOperand(3), - N->getOperand(4), N->getOperand(5) }; - return CurDAG->getTargetNode(Opc, dl, EVT::Other, Ops, 7); - } - case ISD::INTRINSIC_WO_CHAIN: { - unsigned IntNo = cast(N->getOperand(0))->getZExtValue(); - EVT VT = N->getValueType(0); - unsigned Opc = 0; + case Intrinsic::arm_neon_vst3lane: { + unsigned DOpcodes[] = { ARM::VST3LNd8, ARM::VST3LNd16, ARM::VST3LNd32 }; + unsigned QOpcodes0[] = { ARM::VST3LNq16a, ARM::VST3LNq32a }; + unsigned QOpcodes1[] = { ARM::VST3LNq16b, ARM::VST3LNq32b }; + return SelectVLDSTLane(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); + } - // Match intrinsics that return multiple values. - switch (IntNo) { - default: break; - - case Intrinsic::arm_neon_vtrn: - switch (VT.getSimpleVT()) { - default: return NULL; - case EVT::v8i8: Opc = ARM::VTRNd8; break; - case EVT::v4i16: Opc = ARM::VTRNd16; break; - case EVT::v2f32: - case EVT::v2i32: Opc = ARM::VTRNd32; break; - case EVT::v16i8: Opc = ARM::VTRNq8; break; - case EVT::v8i16: Opc = ARM::VTRNq16; break; - case EVT::v4f32: - case EVT::v4i32: Opc = ARM::VTRNq32; break; - } - return CurDAG->getTargetNode(Opc, dl, VT, VT, N->getOperand(1), - N->getOperand(2)); - - case Intrinsic::arm_neon_vuzp: - switch (VT.getSimpleVT()) { - default: return NULL; - case EVT::v8i8: Opc = ARM::VUZPd8; break; - case EVT::v4i16: Opc = ARM::VUZPd16; break; - case EVT::v2f32: - case EVT::v2i32: Opc = ARM::VUZPd32; break; - case EVT::v16i8: Opc = ARM::VUZPq8; break; - case EVT::v8i16: Opc = ARM::VUZPq16; break; - case EVT::v4f32: - case EVT::v4i32: Opc = ARM::VUZPq32; break; - } - return CurDAG->getTargetNode(Opc, dl, VT, VT, N->getOperand(1), - N->getOperand(2)); - - case Intrinsic::arm_neon_vzip: - switch (VT.getSimpleVT()) { - default: return NULL; - case EVT::v8i8: Opc = ARM::VZIPd8; break; - case EVT::v4i16: Opc = ARM::VZIPd16; break; - case EVT::v2f32: - case EVT::v2i32: Opc = ARM::VZIPd32; break; - case EVT::v16i8: Opc = ARM::VZIPq8; break; - case EVT::v8i16: Opc = ARM::VZIPq16; break; - case EVT::v4f32: - case EVT::v4i32: Opc = ARM::VZIPq32; break; - } - return CurDAG->getTargetNode(Opc, dl, VT, VT, N->getOperand(1), - N->getOperand(2)); + case Intrinsic::arm_neon_vst4lane: { + unsigned DOpcodes[] = { ARM::VST4LNd8, ARM::VST4LNd16, ARM::VST4LNd32 }; + unsigned QOpcodes0[] = { ARM::VST4LNq16a, ARM::VST4LNq32a }; + unsigned QOpcodes1[] = { ARM::VST4LNq16b, ARM::VST4LNq32b }; + return SelectVLDSTLane(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); + } } - break; } } - return SelectCode(Op); + return SelectCode(N); } bool ARMDAGToDAGISel:: SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, std::vector &OutOps) { assert(ConstraintCode == 'm' && "unexpected asm memory constraint"); - - SDValue Base, Offset, Opc; - if (!SelectAddrMode2(Op, Op, Base, Offset, Opc)) - return true; - - OutOps.push_back(Base); - OutOps.push_back(Offset); - OutOps.push_back(Opc); + // Require the address to be in a register. That is safe for all ARM + // variants and it is hard to do anything much smarter without knowing + // how the operand is used. + OutOps.push_back(Op); return false; } /// createARMISelDag - This pass converts a legalized DAG into a /// ARM-specific DAG, ready for instruction scheduling. /// -FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM) { - return new ARMDAGToDAGISel(TM); +FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM, + CodeGenOpt::Level OptLevel) { + return new ARMDAGToDAGISel(TM, OptLevel); }