-//===-- X86FastISel.cpp - X86 FastISel implementation ---------------------===//\r
-//\r
-// The LLVM Compiler Infrastructure\r
-//\r
-// This file is distributed under the University of Illinois Open Source\r
-// License. See LICENSE.TXT for details.\r
-//\r
-//===----------------------------------------------------------------------===//\r
-//\r
-// This file defines the X86-specific support for the FastISel class. Much\r
-// of the target-specific code is generated by tablegen in the file\r
-// X86GenFastISel.inc, which is #included here.\r
-//\r
-//===----------------------------------------------------------------------===//\r
-\r
-#include "X86.h"\r
-#include "X86CallingConv.h"\r
-#include "X86InstrBuilder.h"\r
-#include "X86InstrInfo.h"\r
-#include "X86MachineFunctionInfo.h"\r
-#include "X86RegisterInfo.h"\r
-#include "X86Subtarget.h"\r
-#include "X86TargetMachine.h"\r
-#include "llvm/Analysis/BranchProbabilityInfo.h"\r
-#include "llvm/CodeGen/Analysis.h"\r
-#include "llvm/CodeGen/FastISel.h"\r
-#include "llvm/CodeGen/FunctionLoweringInfo.h"\r
-#include "llvm/CodeGen/MachineConstantPool.h"\r
-#include "llvm/CodeGen/MachineFrameInfo.h"\r
-#include "llvm/CodeGen/MachineRegisterInfo.h"\r
-#include "llvm/IR/CallSite.h"\r
-#include "llvm/IR/CallingConv.h"\r
-#include "llvm/IR/DerivedTypes.h"\r
-#include "llvm/IR/GetElementPtrTypeIterator.h"\r
-#include "llvm/IR/GlobalAlias.h"\r
-#include "llvm/IR/GlobalVariable.h"\r
-#include "llvm/IR/Instructions.h"\r
-#include "llvm/IR/IntrinsicInst.h"\r
-#include "llvm/IR/Operator.h"\r
-#include "llvm/Support/ErrorHandling.h"\r
-#include "llvm/Target/TargetOptions.h"\r
-using namespace llvm;\r
-\r
-namespace {\r
-\r
-class X86FastISel final : public FastISel {\r
- /// Subtarget - Keep a pointer to the X86Subtarget around so that we can\r
- /// make the right decision when generating code for different targets.\r
- const X86Subtarget *Subtarget;\r
-\r
- /// X86ScalarSSEf32, X86ScalarSSEf64 - Select between SSE or x87\r
- /// floating point ops.\r
- /// When SSE is available, use it for f32 operations.\r
- /// When SSE2 is available, use it for f64 operations.\r
- bool X86ScalarSSEf64;\r
- bool X86ScalarSSEf32;\r
-\r
-public:\r
- explicit X86FastISel(FunctionLoweringInfo &funcInfo,\r
- const TargetLibraryInfo *libInfo)\r
- : FastISel(funcInfo, libInfo) {\r
- Subtarget = &TM.getSubtarget<X86Subtarget>();\r
- X86ScalarSSEf64 = Subtarget->hasSSE2();\r
- X86ScalarSSEf32 = Subtarget->hasSSE1();\r
- }\r
-\r
- bool fastSelectInstruction(const Instruction *I) override;\r
-\r
- /// \brief The specified machine instr operand is a vreg, and that\r
- /// vreg is being provided by the specified load instruction. If possible,\r
- /// try to fold the load as an operand to the instruction, returning true if\r
- /// possible.\r
- bool tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,\r
- const LoadInst *LI) override;\r
-\r
- bool fastLowerArguments() override;\r
- bool fastLowerCall(CallLoweringInfo &CLI) override;\r
- bool fastLowerIntrinsicCall(const IntrinsicInst *II) override;\r
-\r
-#include "X86GenFastISel.inc"\r
-\r
-private:\r
- bool X86FastEmitCompare(const Value *LHS, const Value *RHS, EVT VT, DebugLoc DL);\r
-\r
- bool X86FastEmitLoad(EVT VT, const X86AddressMode &AM, MachineMemOperand *MMO,\r
- unsigned &ResultReg);\r
-\r
- bool X86FastEmitStore(EVT VT, const Value *Val, const X86AddressMode &AM,\r
- MachineMemOperand *MMO = nullptr, bool Aligned = false);\r
- bool X86FastEmitStore(EVT VT, unsigned ValReg, bool ValIsKill,\r
- const X86AddressMode &AM,\r
- MachineMemOperand *MMO = nullptr, bool Aligned = false);\r
-\r
- bool X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, unsigned Src, EVT SrcVT,\r
- unsigned &ResultReg);\r
-\r
- bool X86SelectAddress(const Value *V, X86AddressMode &AM);\r
- bool X86SelectCallAddress(const Value *V, X86AddressMode &AM);\r
-\r
- bool X86SelectLoad(const Instruction *I);\r
-\r
- bool X86SelectStore(const Instruction *I);\r
-\r
- bool X86SelectRet(const Instruction *I);\r
-\r
- bool X86SelectCmp(const Instruction *I);\r
-\r
- bool X86SelectZExt(const Instruction *I);\r
-\r
- bool X86SelectBranch(const Instruction *I);\r
-\r
- bool X86SelectShift(const Instruction *I);\r
-\r
- bool X86SelectDivRem(const Instruction *I);\r
-\r
- bool X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I);\r
-\r
- bool X86FastEmitSSESelect(MVT RetVT, const Instruction *I);\r
-\r
- bool X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I);\r
-\r
- bool X86SelectSelect(const Instruction *I);\r
-\r
- bool X86SelectTrunc(const Instruction *I);\r
-\r
- bool X86SelectFPExt(const Instruction *I);\r
- bool X86SelectFPTrunc(const Instruction *I);\r
-\r
- const X86InstrInfo *getInstrInfo() const {\r
- return getTargetMachine()->getSubtargetImpl()->getInstrInfo();\r
- }\r
- const X86TargetMachine *getTargetMachine() const {\r
- return static_cast<const X86TargetMachine *>(&TM);\r
- }\r
-\r
- bool handleConstantAddresses(const Value *V, X86AddressMode &AM);\r
-\r
- unsigned X86MaterializeInt(const ConstantInt *CI, MVT VT);\r
- unsigned X86MaterializeFP(const ConstantFP *CFP, MVT VT);\r
- unsigned X86MaterializeGV(const GlobalValue *GV, MVT VT);\r
- unsigned fastMaterializeConstant(const Constant *C) override;\r
-\r
- unsigned fastMaterializeAlloca(const AllocaInst *C) override;\r
-\r
- unsigned fastMaterializeFloatZero(const ConstantFP *CF) override;\r
-\r
- /// isScalarFPTypeInSSEReg - Return true if the specified scalar FP type is\r
- /// computed in an SSE register, not on the X87 floating point stack.\r
- bool isScalarFPTypeInSSEReg(EVT VT) const {\r
- return (VT == MVT::f64 && X86ScalarSSEf64) || // f64 is when SSE2\r
- (VT == MVT::f32 && X86ScalarSSEf32); // f32 is when SSE1\r
- }\r
-\r
- bool isTypeLegal(Type *Ty, MVT &VT, bool AllowI1 = false);\r
-\r
- bool IsMemcpySmall(uint64_t Len);\r
-\r
- bool TryEmitSmallMemcpy(X86AddressMode DestAM,\r
- X86AddressMode SrcAM, uint64_t Len);\r
-\r
- bool foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,\r
- const Value *Cond);\r
-};\r
-\r
-} // end anonymous namespace.\r
-\r
-static std::pair<X86::CondCode, bool>\r
-getX86ConditionCode(CmpInst::Predicate Predicate) {\r
- X86::CondCode CC = X86::COND_INVALID;\r
- bool NeedSwap = false;\r
- switch (Predicate) {\r
- default: break;\r
- // Floating-point Predicates\r
- case CmpInst::FCMP_UEQ: CC = X86::COND_E; break;\r
- case CmpInst::FCMP_OLT: NeedSwap = true; // fall-through\r
- case CmpInst::FCMP_OGT: CC = X86::COND_A; break;\r
- case CmpInst::FCMP_OLE: NeedSwap = true; // fall-through\r
- case CmpInst::FCMP_OGE: CC = X86::COND_AE; break;\r
- case CmpInst::FCMP_UGT: NeedSwap = true; // fall-through\r
- case CmpInst::FCMP_ULT: CC = X86::COND_B; break;\r
- case CmpInst::FCMP_UGE: NeedSwap = true; // fall-through\r
- case CmpInst::FCMP_ULE: CC = X86::COND_BE; break;\r
- case CmpInst::FCMP_ONE: CC = X86::COND_NE; break;\r
- case CmpInst::FCMP_UNO: CC = X86::COND_P; break;\r
- case CmpInst::FCMP_ORD: CC = X86::COND_NP; break;\r
- case CmpInst::FCMP_OEQ: // fall-through\r
- case CmpInst::FCMP_UNE: CC = X86::COND_INVALID; break;\r
-\r
- // Integer Predicates\r
- case CmpInst::ICMP_EQ: CC = X86::COND_E; break;\r
- case CmpInst::ICMP_NE: CC = X86::COND_NE; break;\r
- case CmpInst::ICMP_UGT: CC = X86::COND_A; break;\r
- case CmpInst::ICMP_UGE: CC = X86::COND_AE; break;\r
- case CmpInst::ICMP_ULT: CC = X86::COND_B; break;\r
- case CmpInst::ICMP_ULE: CC = X86::COND_BE; break;\r
- case CmpInst::ICMP_SGT: CC = X86::COND_G; break;\r
- case CmpInst::ICMP_SGE: CC = X86::COND_GE; break;\r
- case CmpInst::ICMP_SLT: CC = X86::COND_L; break;\r
- case CmpInst::ICMP_SLE: CC = X86::COND_LE; break;\r
- }\r
-\r
- return std::make_pair(CC, NeedSwap);\r
-}\r
-\r
-static std::pair<unsigned, bool>\r
-getX86SSEConditionCode(CmpInst::Predicate Predicate) {\r
- unsigned CC;\r
- bool NeedSwap = false;\r
-\r
- // SSE Condition code mapping:\r
- // 0 - EQ\r
- // 1 - LT\r
- // 2 - LE\r
- // 3 - UNORD\r
- // 4 - NEQ\r
- // 5 - NLT\r
- // 6 - NLE\r
- // 7 - ORD\r
- switch (Predicate) {\r
- default: llvm_unreachable("Unexpected predicate");\r
- case CmpInst::FCMP_OEQ: CC = 0; break;\r
- case CmpInst::FCMP_OGT: NeedSwap = true; // fall-through\r
- case CmpInst::FCMP_OLT: CC = 1; break;\r
- case CmpInst::FCMP_OGE: NeedSwap = true; // fall-through\r
- case CmpInst::FCMP_OLE: CC = 2; break;\r
- case CmpInst::FCMP_UNO: CC = 3; break;\r
- case CmpInst::FCMP_UNE: CC = 4; break;\r
- case CmpInst::FCMP_ULE: NeedSwap = true; // fall-through\r
- case CmpInst::FCMP_UGE: CC = 5; break;\r
- case CmpInst::FCMP_ULT: NeedSwap = true; // fall-through\r
- case CmpInst::FCMP_UGT: CC = 6; break;\r
- case CmpInst::FCMP_ORD: CC = 7; break;\r
- case CmpInst::FCMP_UEQ:\r
- case CmpInst::FCMP_ONE: CC = 8; break;\r
- }\r
-\r
- return std::make_pair(CC, NeedSwap);\r
-}\r
-\r
-/// \brief Check if it is possible to fold the condition from the XALU intrinsic\r
-/// into the user. The condition code will only be updated on success.\r
-bool X86FastISel::foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,\r
- const Value *Cond) {\r
- if (!isa<ExtractValueInst>(Cond))\r
- return false;\r
-\r
- const auto *EV = cast<ExtractValueInst>(Cond);\r
- if (!isa<IntrinsicInst>(EV->getAggregateOperand()))\r
- return false;\r
-\r
- const auto *II = cast<IntrinsicInst>(EV->getAggregateOperand());\r
- MVT RetVT;\r
- const Function *Callee = II->getCalledFunction();\r
- Type *RetTy =\r
- cast<StructType>(Callee->getReturnType())->getTypeAtIndex(0U);\r
- if (!isTypeLegal(RetTy, RetVT))\r
- return false;\r
-\r
- if (RetVT != MVT::i32 && RetVT != MVT::i64)\r
- return false;\r
-\r
- X86::CondCode TmpCC;\r
- switch (II->getIntrinsicID()) {\r
- default: return false;\r
- case Intrinsic::sadd_with_overflow:\r
- case Intrinsic::ssub_with_overflow:\r
- case Intrinsic::smul_with_overflow:\r
- case Intrinsic::umul_with_overflow: TmpCC = X86::COND_O; break;\r
- case Intrinsic::uadd_with_overflow:\r
- case Intrinsic::usub_with_overflow: TmpCC = X86::COND_B; break;\r
- }\r
-\r
- // Check if both instructions are in the same basic block.\r
- if (II->getParent() != I->getParent())\r
- return false;\r
-\r
- // Make sure nothing is in the way\r
- BasicBlock::const_iterator Start = I;\r
- BasicBlock::const_iterator End = II;\r
- for (auto Itr = std::prev(Start); Itr != End; --Itr) {\r
- // We only expect extractvalue instructions between the intrinsic and the\r
- // instruction to be selected.\r
- if (!isa<ExtractValueInst>(Itr))\r
- return false;\r
-\r
- // Check that the extractvalue operand comes from the intrinsic.\r
- const auto *EVI = cast<ExtractValueInst>(Itr);\r
- if (EVI->getAggregateOperand() != II)\r
- return false;\r
- }\r
-\r
- CC = TmpCC;\r
- return true;\r
-}\r
-\r
-bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {\r
- EVT evt = TLI.getValueType(Ty, /*HandleUnknown=*/true);\r
- if (evt == MVT::Other || !evt.isSimple())\r
- // Unhandled type. Halt "fast" selection and bail.\r
- return false;\r
-\r
- VT = evt.getSimpleVT();\r
- // For now, require SSE/SSE2 for performing floating-point operations,\r
- // since x87 requires additional work.\r
- if (VT == MVT::f64 && !X86ScalarSSEf64)\r
- return false;\r
- if (VT == MVT::f32 && !X86ScalarSSEf32)\r
- return false;\r
- // Similarly, no f80 support yet.\r
- if (VT == MVT::f80)\r
- return false;\r
- // We only handle legal types. For example, on x86-32 the instruction\r
- // selector contains all of the 64-bit instructions from x86-64,\r
- // under the assumption that i64 won't be used if the target doesn't\r
- // support it.\r
- return (AllowI1 && VT == MVT::i1) || TLI.isTypeLegal(VT);\r
-}\r
-\r
-#include "X86GenCallingConv.inc"\r
-\r
-/// X86FastEmitLoad - Emit a machine instruction to load a value of type VT.\r
-/// The address is either pre-computed, i.e. Ptr, or a GlobalAddress, i.e. GV.\r
-/// Return true and the result register by reference if it is possible.\r
-bool X86FastISel::X86FastEmitLoad(EVT VT, const X86AddressMode &AM,\r
- MachineMemOperand *MMO, unsigned &ResultReg) {\r
- // Get opcode and regclass of the output for the given load instruction.\r
- unsigned Opc = 0;\r
- const TargetRegisterClass *RC = nullptr;\r
- switch (VT.getSimpleVT().SimpleTy) {\r
- default: return false;\r
- case MVT::i1:\r
- case MVT::i8:\r
- Opc = X86::MOV8rm;\r
- RC = &X86::GR8RegClass;\r
- break;\r
- case MVT::i16:\r
- Opc = X86::MOV16rm;\r
- RC = &X86::GR16RegClass;\r
- break;\r
- case MVT::i32:\r
- Opc = X86::MOV32rm;\r
- RC = &X86::GR32RegClass;\r
- break;\r
- case MVT::i64:\r
- // Must be in x86-64 mode.\r
- Opc = X86::MOV64rm;\r
- RC = &X86::GR64RegClass;\r
- break;\r
- case MVT::f32:\r
- if (X86ScalarSSEf32) {\r
- Opc = Subtarget->hasAVX() ? X86::VMOVSSrm : X86::MOVSSrm;\r
- RC = &X86::FR32RegClass;\r
- } else {\r
- Opc = X86::LD_Fp32m;\r
- RC = &X86::RFP32RegClass;\r
- }\r
- break;\r
- case MVT::f64:\r
- if (X86ScalarSSEf64) {\r
- Opc = Subtarget->hasAVX() ? X86::VMOVSDrm : X86::MOVSDrm;\r
- RC = &X86::FR64RegClass;\r
- } else {\r
- Opc = X86::LD_Fp64m;\r
- RC = &X86::RFP64RegClass;\r
- }\r
- break;\r
- case MVT::f80:\r
- // No f80 support yet.\r
- return false;\r
- }\r
-\r
- ResultReg = createResultReg(RC);\r
- MachineInstrBuilder MIB =\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg);\r
- addFullAddress(MIB, AM);\r
- if (MMO)\r
- MIB->addMemOperand(*FuncInfo.MF, MMO);\r
- return true;\r
-}\r
-\r
-/// X86FastEmitStore - Emit a machine instruction to store a value Val of\r
-/// type VT. The address is either pre-computed, consisted of a base ptr, Ptr\r
-/// and a displacement offset, or a GlobalAddress,\r
-/// i.e. V. Return true if it is possible.\r
-bool X86FastISel::X86FastEmitStore(EVT VT, unsigned ValReg, bool ValIsKill,\r
- const X86AddressMode &AM,\r
- MachineMemOperand *MMO, bool Aligned) {\r
- // Get opcode and regclass of the output for the given store instruction.\r
- unsigned Opc = 0;\r
- switch (VT.getSimpleVT().SimpleTy) {\r
- case MVT::f80: // No f80 support yet.\r
- default: return false;\r
- case MVT::i1: {\r
- // Mask out all but lowest bit.\r
- unsigned AndResult = createResultReg(&X86::GR8RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(X86::AND8ri), AndResult)\r
- .addReg(ValReg, getKillRegState(ValIsKill)).addImm(1);\r
- ValReg = AndResult;\r
- }\r
- // FALLTHROUGH, handling i1 as i8.\r
- case MVT::i8: Opc = X86::MOV8mr; break;\r
- case MVT::i16: Opc = X86::MOV16mr; break;\r
- case MVT::i32: Opc = X86::MOV32mr; break;\r
- case MVT::i64: Opc = X86::MOV64mr; break; // Must be in x86-64 mode.\r
- case MVT::f32:\r
- Opc = X86ScalarSSEf32 ?\r
- (Subtarget->hasAVX() ? X86::VMOVSSmr : X86::MOVSSmr) : X86::ST_Fp32m;\r
- break;\r
- case MVT::f64:\r
- Opc = X86ScalarSSEf64 ?\r
- (Subtarget->hasAVX() ? X86::VMOVSDmr : X86::MOVSDmr) : X86::ST_Fp64m;\r
- break;\r
- case MVT::v4f32:\r
- if (Aligned)\r
- Opc = Subtarget->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr;\r
- else\r
- Opc = Subtarget->hasAVX() ? X86::VMOVUPSmr : X86::MOVUPSmr;\r
- break;\r
- case MVT::v2f64:\r
- if (Aligned)\r
- Opc = Subtarget->hasAVX() ? X86::VMOVAPDmr : X86::MOVAPDmr;\r
- else\r
- Opc = Subtarget->hasAVX() ? X86::VMOVUPDmr : X86::MOVUPDmr;\r
- break;\r
- case MVT::v4i32:\r
- case MVT::v2i64:\r
- case MVT::v8i16:\r
- case MVT::v16i8:\r
- if (Aligned)\r
- Opc = Subtarget->hasAVX() ? X86::VMOVDQAmr : X86::MOVDQAmr;\r
- else\r
- Opc = Subtarget->hasAVX() ? X86::VMOVDQUmr : X86::MOVDQUmr;\r
- break;\r
- }\r
-\r
- MachineInstrBuilder MIB =\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc));\r
- addFullAddress(MIB, AM).addReg(ValReg, getKillRegState(ValIsKill));\r
- if (MMO)\r
- MIB->addMemOperand(*FuncInfo.MF, MMO);\r
-\r
- return true;\r
-}\r
-\r
-bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val,\r
- const X86AddressMode &AM,\r
- MachineMemOperand *MMO, bool Aligned) {\r
- // Handle 'null' like i32/i64 0.\r
- if (isa<ConstantPointerNull>(Val))\r
- Val = Constant::getNullValue(DL.getIntPtrType(Val->getContext()));\r
-\r
- // If this is a store of a simple constant, fold the constant into the store.\r
- if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val)) {\r
- unsigned Opc = 0;\r
- bool Signed = true;\r
- switch (VT.getSimpleVT().SimpleTy) {\r
- default: break;\r
- case MVT::i1: Signed = false; // FALLTHROUGH to handle as i8.\r
- case MVT::i8: Opc = X86::MOV8mi; break;\r
- case MVT::i16: Opc = X86::MOV16mi; break;\r
- case MVT::i32: Opc = X86::MOV32mi; break;\r
- case MVT::i64:\r
- // Must be a 32-bit sign extended value.\r
- if (isInt<32>(CI->getSExtValue()))\r
- Opc = X86::MOV64mi32;\r
- break;\r
- }\r
-\r
- if (Opc) {\r
- MachineInstrBuilder MIB =\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc));\r
- addFullAddress(MIB, AM).addImm(Signed ? (uint64_t) CI->getSExtValue()\r
- : CI->getZExtValue());\r
- if (MMO)\r
- MIB->addMemOperand(*FuncInfo.MF, MMO);\r
- return true;\r
- }\r
- }\r
-\r
- unsigned ValReg = getRegForValue(Val);\r
- if (ValReg == 0)\r
- return false;\r
-\r
- bool ValKill = hasTrivialKill(Val);\r
- return X86FastEmitStore(VT, ValReg, ValKill, AM, MMO, Aligned);\r
-}\r
-\r
-/// X86FastEmitExtend - Emit a machine instruction to extend a value Src of\r
-/// type SrcVT to type DstVT using the specified extension opcode Opc (e.g.\r
-/// ISD::SIGN_EXTEND).\r
-bool X86FastISel::X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT,\r
- unsigned Src, EVT SrcVT,\r
- unsigned &ResultReg) {\r
- unsigned RR = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), Opc,\r
- Src, /*TODO: Kill=*/false);\r
- if (RR == 0)\r
- return false;\r
-\r
- ResultReg = RR;\r
- return true;\r
-}\r
-\r
-bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {\r
- // Handle constant address.\r
- if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {\r
- // Can't handle alternate code models yet.\r
- if (TM.getCodeModel() != CodeModel::Small)\r
- return false;\r
-\r
- // Can't handle TLS yet.\r
- if (GV->isThreadLocal())\r
- return false;\r
-\r
- // RIP-relative addresses can't have additional register operands, so if\r
- // we've already folded stuff into the addressing mode, just force the\r
- // global value into its own register, which we can use as the basereg.\r
- if (!Subtarget->isPICStyleRIPRel() ||\r
- (AM.Base.Reg == 0 && AM.IndexReg == 0)) {\r
- // Okay, we've committed to selecting this global. Set up the address.\r
- AM.GV = GV;\r
-\r
- // Allow the subtarget to classify the global.\r
- unsigned char GVFlags = Subtarget->ClassifyGlobalReference(GV, TM);\r
-\r
- // If this reference is relative to the pic base, set it now.\r
- if (isGlobalRelativeToPICBase(GVFlags)) {\r
- // FIXME: How do we know Base.Reg is free??\r
- AM.Base.Reg = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);\r
- }\r
-\r
- // Unless the ABI requires an extra load, return a direct reference to\r
- // the global.\r
- if (!isGlobalStubReference(GVFlags)) {\r
- if (Subtarget->isPICStyleRIPRel()) {\r
- // Use rip-relative addressing if we can. Above we verified that the\r
- // base and index registers are unused.\r
- assert(AM.Base.Reg == 0 && AM.IndexReg == 0);\r
- AM.Base.Reg = X86::RIP;\r
- }\r
- AM.GVOpFlags = GVFlags;\r
- return true;\r
- }\r
-\r
- // Ok, we need to do a load from a stub. If we've already loaded from\r
- // this stub, reuse the loaded pointer, otherwise emit the load now.\r
- DenseMap<const Value *, unsigned>::iterator I = LocalValueMap.find(V);\r
- unsigned LoadReg;\r
- if (I != LocalValueMap.end() && I->second != 0) {\r
- LoadReg = I->second;\r
- } else {\r
- // Issue load from stub.\r
- unsigned Opc = 0;\r
- const TargetRegisterClass *RC = nullptr;\r
- X86AddressMode StubAM;\r
- StubAM.Base.Reg = AM.Base.Reg;\r
- StubAM.GV = GV;\r
- StubAM.GVOpFlags = GVFlags;\r
-\r
- // Prepare for inserting code in the local-value area.\r
- SavePoint SaveInsertPt = enterLocalValueArea();\r
-\r
- if (TLI.getPointerTy() == MVT::i64) {\r
- Opc = X86::MOV64rm;\r
- RC = &X86::GR64RegClass;\r
-\r
- if (Subtarget->isPICStyleRIPRel())\r
- StubAM.Base.Reg = X86::RIP;\r
- } else {\r
- Opc = X86::MOV32rm;\r
- RC = &X86::GR32RegClass;\r
- }\r
-\r
- LoadReg = createResultReg(RC);\r
- MachineInstrBuilder LoadMI =\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), LoadReg);\r
- addFullAddress(LoadMI, StubAM);\r
-\r
- // Ok, back to normal mode.\r
- leaveLocalValueArea(SaveInsertPt);\r
-\r
- // Prevent loading GV stub multiple times in same MBB.\r
- LocalValueMap[V] = LoadReg;\r
- }\r
-\r
- // Now construct the final address. Note that the Disp, Scale,\r
- // and Index values may already be set here.\r
- AM.Base.Reg = LoadReg;\r
- AM.GV = nullptr;\r
- return true;\r
- }\r
- }\r
-\r
- // If all else fails, try to materialize the value in a register.\r
- if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {\r
- if (AM.Base.Reg == 0) {\r
- AM.Base.Reg = getRegForValue(V);\r
- return AM.Base.Reg != 0;\r
- }\r
- if (AM.IndexReg == 0) {\r
- assert(AM.Scale == 1 && "Scale with no index!");\r
- AM.IndexReg = getRegForValue(V);\r
- return AM.IndexReg != 0;\r
- }\r
- }\r
-\r
- return false;\r
-}\r
-\r
-/// X86SelectAddress - Attempt to fill in an address from the given value.\r
-///\r
-bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {\r
- SmallVector<const Value *, 32> GEPs;\r
-redo_gep:\r
- const User *U = nullptr;\r
- unsigned Opcode = Instruction::UserOp1;\r
- if (const Instruction *I = dyn_cast<Instruction>(V)) {\r
- // Don't walk into other basic blocks; it's possible we haven't\r
- // visited them yet, so the instructions may not yet be assigned\r
- // virtual registers.\r
- if (FuncInfo.StaticAllocaMap.count(static_cast<const AllocaInst *>(V)) ||\r
- FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) {\r
- Opcode = I->getOpcode();\r
- U = I;\r
- }\r
- } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(V)) {\r
- Opcode = C->getOpcode();\r
- U = C;\r
- }\r
-\r
- if (PointerType *Ty = dyn_cast<PointerType>(V->getType()))\r
- if (Ty->getAddressSpace() > 255)\r
- // Fast instruction selection doesn't support the special\r
- // address spaces.\r
- return false;\r
-\r
- switch (Opcode) {\r
- default: break;\r
- case Instruction::BitCast:\r
- // Look past bitcasts.\r
- return X86SelectAddress(U->getOperand(0), AM);\r
-\r
- case Instruction::IntToPtr:\r
- // Look past no-op inttoptrs.\r
- if (TLI.getValueType(U->getOperand(0)->getType()) == TLI.getPointerTy())\r
- return X86SelectAddress(U->getOperand(0), AM);\r
- break;\r
-\r
- case Instruction::PtrToInt:\r
- // Look past no-op ptrtoints.\r
- if (TLI.getValueType(U->getType()) == TLI.getPointerTy())\r
- return X86SelectAddress(U->getOperand(0), AM);\r
- break;\r
-\r
- case Instruction::Alloca: {\r
- // Do static allocas.\r
- const AllocaInst *A = cast<AllocaInst>(V);\r
- DenseMap<const AllocaInst *, int>::iterator SI =\r
- FuncInfo.StaticAllocaMap.find(A);\r
- if (SI != FuncInfo.StaticAllocaMap.end()) {\r
- AM.BaseType = X86AddressMode::FrameIndexBase;\r
- AM.Base.FrameIndex = SI->second;\r
- return true;\r
- }\r
- break;\r
- }\r
-\r
- case Instruction::Add: {\r
- // Adds of constants are common and easy enough.\r
- if (const ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {\r
- uint64_t Disp = (int32_t)AM.Disp + (uint64_t)CI->getSExtValue();\r
- // They have to fit in the 32-bit signed displacement field though.\r
- if (isInt<32>(Disp)) {\r
- AM.Disp = (uint32_t)Disp;\r
- return X86SelectAddress(U->getOperand(0), AM);\r
- }\r
- }\r
- break;\r
- }\r
-\r
- case Instruction::GetElementPtr: {\r
- X86AddressMode SavedAM = AM;\r
-\r
- // Pattern-match simple GEPs.\r
- uint64_t Disp = (int32_t)AM.Disp;\r
- unsigned IndexReg = AM.IndexReg;\r
- unsigned Scale = AM.Scale;\r
- gep_type_iterator GTI = gep_type_begin(U);\r
- // Iterate through the indices, folding what we can. Constants can be\r
- // folded, and one dynamic index can be handled, if the scale is supported.\r
- for (User::const_op_iterator i = U->op_begin() + 1, e = U->op_end();\r
- i != e; ++i, ++GTI) {\r
- const Value *Op = *i;\r
- if (StructType *STy = dyn_cast<StructType>(*GTI)) {\r
- const StructLayout *SL = DL.getStructLayout(STy);\r
- Disp += SL->getElementOffset(cast<ConstantInt>(Op)->getZExtValue());\r
- continue;\r
- }\r
-\r
- // A array/variable index is always of the form i*S where S is the\r
- // constant scale size. See if we can push the scale into immediates.\r
- uint64_t S = DL.getTypeAllocSize(GTI.getIndexedType());\r
- for (;;) {\r
- if (const ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {\r
- // Constant-offset addressing.\r
- Disp += CI->getSExtValue() * S;\r
- break;\r
- }\r
- if (canFoldAddIntoGEP(U, Op)) {\r
- // A compatible add with a constant operand. Fold the constant.\r
- ConstantInt *CI =\r
- cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1));\r
- Disp += CI->getSExtValue() * S;\r
- // Iterate on the other operand.\r
- Op = cast<AddOperator>(Op)->getOperand(0);\r
- continue;\r
- }\r
- if (IndexReg == 0 &&\r
- (!AM.GV || !Subtarget->isPICStyleRIPRel()) &&\r
- (S == 1 || S == 2 || S == 4 || S == 8)) {\r
- // Scaled-index addressing.\r
- Scale = S;\r
- IndexReg = getRegForGEPIndex(Op).first;\r
- if (IndexReg == 0)\r
- return false;\r
- break;\r
- }\r
- // Unsupported.\r
- goto unsupported_gep;\r
- }\r
- }\r
-\r
- // Check for displacement overflow.\r
- if (!isInt<32>(Disp))\r
- break;\r
-\r
- AM.IndexReg = IndexReg;\r
- AM.Scale = Scale;\r
- AM.Disp = (uint32_t)Disp;\r
- GEPs.push_back(V);\r
-\r
- if (const GetElementPtrInst *GEP =\r
- dyn_cast<GetElementPtrInst>(U->getOperand(0))) {\r
- // Ok, the GEP indices were covered by constant-offset and scaled-index\r
- // addressing. Update the address state and move on to examining the base.\r
- V = GEP;\r
- goto redo_gep;\r
- } else if (X86SelectAddress(U->getOperand(0), AM)) {\r
- return true;\r
- }\r
-\r
- // If we couldn't merge the gep value into this addr mode, revert back to\r
- // our address and just match the value instead of completely failing.\r
- AM = SavedAM;\r
-\r
- for (SmallVectorImpl<const Value *>::reverse_iterator\r
- I = GEPs.rbegin(), E = GEPs.rend(); I != E; ++I)\r
- if (handleConstantAddresses(*I, AM))\r
- return true;\r
-\r
- return false;\r
- unsupported_gep:\r
- // Ok, the GEP indices weren't all covered.\r
- break;\r
- }\r
- }\r
-\r
- return handleConstantAddresses(V, AM);\r
-}\r
-\r
-/// X86SelectCallAddress - Attempt to fill in an address from the given value.\r
-///\r
-bool X86FastISel::X86SelectCallAddress(const Value *V, X86AddressMode &AM) {\r
- const User *U = nullptr;\r
- unsigned Opcode = Instruction::UserOp1;\r
- const Instruction *I = dyn_cast<Instruction>(V);\r
- // Record if the value is defined in the same basic block.\r
- //\r
- // This information is crucial to know whether or not folding an\r
- // operand is valid.\r
- // Indeed, FastISel generates or reuses a virtual register for all\r
- // operands of all instructions it selects. Obviously, the definition and\r
- // its uses must use the same virtual register otherwise the produced\r
- // code is incorrect.\r
- // Before instruction selection, FunctionLoweringInfo::set sets the virtual\r
- // registers for values that are alive across basic blocks. This ensures\r
- // that the values are consistently set between across basic block, even\r
- // if different instruction selection mechanisms are used (e.g., a mix of\r
- // SDISel and FastISel).\r
- // For values local to a basic block, the instruction selection process\r
- // generates these virtual registers with whatever method is appropriate\r
- // for its needs. In particular, FastISel and SDISel do not share the way\r
- // local virtual registers are set.\r
- // Therefore, this is impossible (or at least unsafe) to share values\r
- // between basic blocks unless they use the same instruction selection\r
- // method, which is not guarantee for X86.\r
- // Moreover, things like hasOneUse could not be used accurately, if we\r
- // allow to reference values across basic blocks whereas they are not\r
- // alive across basic blocks initially.\r
- bool InMBB = true;\r
- if (I) {\r
- Opcode = I->getOpcode();\r
- U = I;\r
- InMBB = I->getParent() == FuncInfo.MBB->getBasicBlock();\r
- } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(V)) {\r
- Opcode = C->getOpcode();\r
- U = C;\r
- }\r
-\r
- switch (Opcode) {\r
- default: break;\r
- case Instruction::BitCast:\r
- // Look past bitcasts if its operand is in the same BB.\r
- if (InMBB)\r
- return X86SelectCallAddress(U->getOperand(0), AM);\r
- break;\r
-\r
- case Instruction::IntToPtr:\r
- // Look past no-op inttoptrs if its operand is in the same BB.\r
- if (InMBB &&\r
- TLI.getValueType(U->getOperand(0)->getType()) == TLI.getPointerTy())\r
- return X86SelectCallAddress(U->getOperand(0), AM);\r
- break;\r
-\r
- case Instruction::PtrToInt:\r
- // Look past no-op ptrtoints if its operand is in the same BB.\r
- if (InMBB &&\r
- TLI.getValueType(U->getType()) == TLI.getPointerTy())\r
- return X86SelectCallAddress(U->getOperand(0), AM);\r
- break;\r
- }\r
-\r
- // Handle constant address.\r
- if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {\r
- // Can't handle alternate code models yet.\r
- if (TM.getCodeModel() != CodeModel::Small)\r
- return false;\r
-\r
- // RIP-relative addresses can't have additional register operands.\r
- if (Subtarget->isPICStyleRIPRel() &&\r
- (AM.Base.Reg != 0 || AM.IndexReg != 0))\r
- return false;\r
-\r
- // Can't handle DLL Import.\r
- if (GV->hasDLLImportStorageClass())\r
- return false;\r
-\r
- // Can't handle TLS.\r
- if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV))\r
- if (GVar->isThreadLocal())\r
- return false;\r
-\r
- // Okay, we've committed to selecting this global. Set up the basic address.\r
- AM.GV = GV;\r
-\r
- // No ABI requires an extra load for anything other than DLLImport, which\r
- // we rejected above. Return a direct reference to the global.\r
- if (Subtarget->isPICStyleRIPRel()) {\r
- // Use rip-relative addressing if we can. Above we verified that the\r
- // base and index registers are unused.\r
- assert(AM.Base.Reg == 0 && AM.IndexReg == 0);\r
- AM.Base.Reg = X86::RIP;\r
- } else if (Subtarget->isPICStyleStubPIC()) {\r
- AM.GVOpFlags = X86II::MO_PIC_BASE_OFFSET;\r
- } else if (Subtarget->isPICStyleGOT()) {\r
- AM.GVOpFlags = X86II::MO_GOTOFF;\r
- }\r
-\r
- return true;\r
- }\r
-\r
- // If all else fails, try to materialize the value in a register.\r
- if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {\r
- if (AM.Base.Reg == 0) {\r
- AM.Base.Reg = getRegForValue(V);\r
- return AM.Base.Reg != 0;\r
- }\r
- if (AM.IndexReg == 0) {\r
- assert(AM.Scale == 1 && "Scale with no index!");\r
- AM.IndexReg = getRegForValue(V);\r
- return AM.IndexReg != 0;\r
- }\r
- }\r
-\r
- return false;\r
-}\r
-\r
-\r
-/// X86SelectStore - Select and emit code to implement store instructions.\r
-bool X86FastISel::X86SelectStore(const Instruction *I) {\r
- // Atomic stores need special handling.\r
- const StoreInst *S = cast<StoreInst>(I);\r
-\r
- if (S->isAtomic())\r
- return false;\r
-\r
- const Value *Val = S->getValueOperand();\r
- const Value *Ptr = S->getPointerOperand();\r
-\r
- MVT VT;\r
- if (!isTypeLegal(Val->getType(), VT, /*AllowI1=*/true))\r
- return false;\r
-\r
- unsigned Alignment = S->getAlignment();\r
- unsigned ABIAlignment = DL.getABITypeAlignment(Val->getType());\r
- if (Alignment == 0) // Ensure that codegen never sees alignment 0\r
- Alignment = ABIAlignment;\r
- bool Aligned = Alignment >= ABIAlignment;\r
-\r
- X86AddressMode AM;\r
- if (!X86SelectAddress(Ptr, AM))\r
- return false;\r
-\r
- return X86FastEmitStore(VT, Val, AM, createMachineMemOperandFor(I), Aligned);\r
-}\r
-\r
-/// X86SelectRet - Select and emit code to implement ret instructions.\r
-bool X86FastISel::X86SelectRet(const Instruction *I) {\r
- const ReturnInst *Ret = cast<ReturnInst>(I);\r
- const Function &F = *I->getParent()->getParent();\r
- const X86MachineFunctionInfo *X86MFInfo =\r
- FuncInfo.MF->getInfo<X86MachineFunctionInfo>();\r
-\r
- if (!FuncInfo.CanLowerReturn)\r
- return false;\r
-\r
- CallingConv::ID CC = F.getCallingConv();\r
- if (CC != CallingConv::C &&\r
- CC != CallingConv::Fast &&\r
- CC != CallingConv::X86_FastCall &&\r
- CC != CallingConv::X86_64_SysV)\r
- return false;\r
-\r
- if (Subtarget->isCallingConvWin64(CC))\r
- return false;\r
-\r
- // Don't handle popping bytes on return for now.\r
- if (X86MFInfo->getBytesToPopOnReturn() != 0)\r
- return false;\r
-\r
- // fastcc with -tailcallopt is intended to provide a guaranteed\r
- // tail call optimization. Fastisel doesn't know how to do that.\r
- if (CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt)\r
- return false;\r
-\r
- // Let SDISel handle vararg functions.\r
- if (F.isVarArg())\r
- return false;\r
-\r
- // Build a list of return value registers.\r
- SmallVector<unsigned, 4> RetRegs;\r
-\r
- if (Ret->getNumOperands() > 0) {\r
- SmallVector<ISD::OutputArg, 4> Outs;\r
- GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI);\r
-\r
- // Analyze operands of the call, assigning locations to each operand.\r
- SmallVector<CCValAssign, 16> ValLocs;\r
- CCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, ValLocs, I->getContext());\r
- CCInfo.AnalyzeReturn(Outs, RetCC_X86);\r
-\r
- const Value *RV = Ret->getOperand(0);\r
- unsigned Reg = getRegForValue(RV);\r
- if (Reg == 0)\r
- return false;\r
-\r
- // Only handle a single return value for now.\r
- if (ValLocs.size() != 1)\r
- return false;\r
-\r
- CCValAssign &VA = ValLocs[0];\r
-\r
- // Don't bother handling odd stuff for now.\r
- if (VA.getLocInfo() != CCValAssign::Full)\r
- return false;\r
- // Only handle register returns for now.\r
- if (!VA.isRegLoc())\r
- return false;\r
-\r
- // The calling-convention tables for x87 returns don't tell\r
- // the whole story.\r
- if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)\r
- return false;\r
-\r
- unsigned SrcReg = Reg + VA.getValNo();\r
- EVT SrcVT = TLI.getValueType(RV->getType());\r
- EVT DstVT = VA.getValVT();\r
- // Special handling for extended integers.\r
- if (SrcVT != DstVT) {\r
- if (SrcVT != MVT::i1 && SrcVT != MVT::i8 && SrcVT != MVT::i16)\r
- return false;\r
-\r
- if (!Outs[0].Flags.isZExt() && !Outs[0].Flags.isSExt())\r
- return false;\r
-\r
- assert(DstVT == MVT::i32 && "X86 should always ext to i32");\r
-\r
- if (SrcVT == MVT::i1) {\r
- if (Outs[0].Flags.isSExt())\r
- return false;\r
- SrcReg = fastEmitZExtFromI1(MVT::i8, SrcReg, /*TODO: Kill=*/false);\r
- SrcVT = MVT::i8;\r
- }\r
- unsigned Op = Outs[0].Flags.isZExt() ? ISD::ZERO_EXTEND :\r
- ISD::SIGN_EXTEND;\r
- SrcReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), Op,\r
- SrcReg, /*TODO: Kill=*/false);\r
- }\r
-\r
- // Make the copy.\r
- unsigned DstReg = VA.getLocReg();\r
- const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);\r
- // Avoid a cross-class copy. This is very unlikely.\r
- if (!SrcRC->contains(DstReg))\r
- return false;\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::COPY), DstReg).addReg(SrcReg);\r
-\r
- // Add register to return instruction.\r
- RetRegs.push_back(VA.getLocReg());\r
- }\r
-\r
- // The x86-64 ABI for returning structs by value requires that we copy\r
- // the sret argument into %rax for the return. We saved the argument into\r
- // a virtual register in the entry block, so now we copy the value out\r
- // and into %rax. We also do the same with %eax for Win32.\r
- if (F.hasStructRetAttr() &&\r
- (Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC())) {\r
- unsigned Reg = X86MFInfo->getSRetReturnReg();\r
- assert(Reg &&\r
- "SRetReturnReg should have been set in LowerFormalArguments()!");\r
- unsigned RetReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::COPY), RetReg).addReg(Reg);\r
- RetRegs.push_back(RetReg);\r
- }\r
-\r
- // Now emit the RET.\r
- MachineInstrBuilder MIB =\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Subtarget->is64Bit() ? X86::RETQ : X86::RETL));\r
- for (unsigned i = 0, e = RetRegs.size(); i != e; ++i)\r
- MIB.addReg(RetRegs[i], RegState::Implicit);\r
- return true;\r
-}\r
-\r
-/// X86SelectLoad - Select and emit code to implement load instructions.\r
-///\r
-bool X86FastISel::X86SelectLoad(const Instruction *I) {\r
- const LoadInst *LI = cast<LoadInst>(I);\r
-\r
- // Atomic loads need special handling.\r
- if (LI->isAtomic())\r
- return false;\r
-\r
- MVT VT;\r
- if (!isTypeLegal(LI->getType(), VT, /*AllowI1=*/true))\r
- return false;\r
-\r
- const Value *Ptr = LI->getPointerOperand();\r
-\r
- X86AddressMode AM;\r
- if (!X86SelectAddress(Ptr, AM))\r
- return false;\r
-\r
- unsigned ResultReg = 0;\r
- if (!X86FastEmitLoad(VT, AM, createMachineMemOperandFor(LI), ResultReg))\r
- return false;\r
-\r
- updateValueMap(I, ResultReg);\r
- return true;\r
-}\r
-\r
-static unsigned X86ChooseCmpOpcode(EVT VT, const X86Subtarget *Subtarget) {\r
- bool HasAVX = Subtarget->hasAVX();\r
- bool X86ScalarSSEf32 = Subtarget->hasSSE1();\r
- bool X86ScalarSSEf64 = Subtarget->hasSSE2();\r
-\r
- switch (VT.getSimpleVT().SimpleTy) {\r
- default: return 0;\r
- case MVT::i8: return X86::CMP8rr;\r
- case MVT::i16: return X86::CMP16rr;\r
- case MVT::i32: return X86::CMP32rr;\r
- case MVT::i64: return X86::CMP64rr;\r
- case MVT::f32:\r
- return X86ScalarSSEf32 ? (HasAVX ? X86::VUCOMISSrr : X86::UCOMISSrr) : 0;\r
- case MVT::f64:\r
- return X86ScalarSSEf64 ? (HasAVX ? X86::VUCOMISDrr : X86::UCOMISDrr) : 0;\r
- }\r
-}\r
-\r
-/// X86ChooseCmpImmediateOpcode - If we have a comparison with RHS as the RHS\r
-/// of the comparison, return an opcode that works for the compare (e.g.\r
-/// CMP32ri) otherwise return 0.\r
-static unsigned X86ChooseCmpImmediateOpcode(EVT VT, const ConstantInt *RHSC) {\r
- switch (VT.getSimpleVT().SimpleTy) {\r
- // Otherwise, we can't fold the immediate into this comparison.\r
- default: return 0;\r
- case MVT::i8: return X86::CMP8ri;\r
- case MVT::i16: return X86::CMP16ri;\r
- case MVT::i32: return X86::CMP32ri;\r
- case MVT::i64:\r
- // 64-bit comparisons are only valid if the immediate fits in a 32-bit sext\r
- // field.\r
- if ((int)RHSC->getSExtValue() == RHSC->getSExtValue())\r
- return X86::CMP64ri32;\r
- return 0;\r
- }\r
-}\r
-\r
-bool X86FastISel::X86FastEmitCompare(const Value *Op0, const Value *Op1,\r
- EVT VT, DebugLoc CurDbgLoc) {\r
- unsigned Op0Reg = getRegForValue(Op0);\r
- if (Op0Reg == 0) return false;\r
-\r
- // Handle 'null' like i32/i64 0.\r
- if (isa<ConstantPointerNull>(Op1))\r
- Op1 = Constant::getNullValue(DL.getIntPtrType(Op0->getContext()));\r
-\r
- // We have two options: compare with register or immediate. If the RHS of\r
- // the compare is an immediate that we can fold into this compare, use\r
- // CMPri, otherwise use CMPrr.\r
- if (const ConstantInt *Op1C = dyn_cast<ConstantInt>(Op1)) {\r
- if (unsigned CompareImmOpc = X86ChooseCmpImmediateOpcode(VT, Op1C)) {\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurDbgLoc, TII.get(CompareImmOpc))\r
- .addReg(Op0Reg)\r
- .addImm(Op1C->getSExtValue());\r
- return true;\r
- }\r
- }\r
-\r
- unsigned CompareOpc = X86ChooseCmpOpcode(VT, Subtarget);\r
- if (CompareOpc == 0) return false;\r
-\r
- unsigned Op1Reg = getRegForValue(Op1);\r
- if (Op1Reg == 0) return false;\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurDbgLoc, TII.get(CompareOpc))\r
- .addReg(Op0Reg)\r
- .addReg(Op1Reg);\r
-\r
- return true;\r
-}\r
-\r
-bool X86FastISel::X86SelectCmp(const Instruction *I) {\r
- const CmpInst *CI = cast<CmpInst>(I);\r
-\r
- MVT VT;\r
- if (!isTypeLegal(I->getOperand(0)->getType(), VT))\r
- return false;\r
-\r
- // Try to optimize or fold the cmp.\r
- CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);\r
- unsigned ResultReg = 0;\r
- switch (Predicate) {\r
- default: break;\r
- case CmpInst::FCMP_FALSE: {\r
- ResultReg = createResultReg(&X86::GR32RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV32r0),\r
- ResultReg);\r
- ResultReg = fastEmitInst_extractsubreg(MVT::i8, ResultReg, /*Kill=*/true,\r
- X86::sub_8bit);\r
- if (!ResultReg)\r
- return false;\r
- break;\r
- }\r
- case CmpInst::FCMP_TRUE: {\r
- ResultReg = createResultReg(&X86::GR8RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV8ri),\r
- ResultReg).addImm(1);\r
- break;\r
- }\r
- }\r
-\r
- if (ResultReg) {\r
- updateValueMap(I, ResultReg);\r
- return true;\r
- }\r
-\r
- const Value *LHS = CI->getOperand(0);\r
- const Value *RHS = CI->getOperand(1);\r
-\r
- // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.\r
- // We don't have to materialize a zero constant for this case and can just use\r
- // %x again on the RHS.\r
- if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {\r
- const auto *RHSC = dyn_cast<ConstantFP>(RHS);\r
- if (RHSC && RHSC->isNullValue())\r
- RHS = LHS;\r
- }\r
-\r
- // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.\r
- static unsigned SETFOpcTable[2][3] = {\r
- { X86::SETEr, X86::SETNPr, X86::AND8rr },\r
- { X86::SETNEr, X86::SETPr, X86::OR8rr }\r
- };\r
- unsigned *SETFOpc = nullptr;\r
- switch (Predicate) {\r
- default: break;\r
- case CmpInst::FCMP_OEQ: SETFOpc = &SETFOpcTable[0][0]; break;\r
- case CmpInst::FCMP_UNE: SETFOpc = &SETFOpcTable[1][0]; break;\r
- }\r
-\r
- ResultReg = createResultReg(&X86::GR8RegClass);\r
- if (SETFOpc) {\r
- if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc()))\r
- return false;\r
-\r
- unsigned FlagReg1 = createResultReg(&X86::GR8RegClass);\r
- unsigned FlagReg2 = createResultReg(&X86::GR8RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[0]),\r
- FlagReg1);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[1]),\r
- FlagReg2);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[2]),\r
- ResultReg).addReg(FlagReg1).addReg(FlagReg2);\r
- updateValueMap(I, ResultReg);\r
- return true;\r
- }\r
-\r
- X86::CondCode CC;\r
- bool SwapArgs;\r
- std::tie(CC, SwapArgs) = getX86ConditionCode(Predicate);\r
- assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");\r
- unsigned Opc = X86::getSETFromCond(CC);\r
-\r
- if (SwapArgs)\r
- std::swap(LHS, RHS);\r
-\r
- // Emit a compare of LHS/RHS.\r
- if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc()))\r
- return false;\r
-\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg);\r
- updateValueMap(I, ResultReg);\r
- return true;\r
-}\r
-\r
-bool X86FastISel::X86SelectZExt(const Instruction *I) {\r
- EVT DstVT = TLI.getValueType(I->getType());\r
- if (!TLI.isTypeLegal(DstVT))\r
- return false;\r
-\r
- unsigned ResultReg = getRegForValue(I->getOperand(0));\r
- if (ResultReg == 0)\r
- return false;\r
-\r
- // Handle zero-extension from i1 to i8, which is common.\r
- MVT SrcVT = TLI.getSimpleValueType(I->getOperand(0)->getType());\r
- if (SrcVT.SimpleTy == MVT::i1) {\r
- // Set the high bits to zero.\r
- ResultReg = fastEmitZExtFromI1(MVT::i8, ResultReg, /*TODO: Kill=*/false);\r
- SrcVT = MVT::i8;\r
-\r
- if (ResultReg == 0)\r
- return false;\r
- }\r
-\r
- if (DstVT == MVT::i64) {\r
- // Handle extension to 64-bits via sub-register shenanigans.\r
- unsigned MovInst;\r
-\r
- switch (SrcVT.SimpleTy) {\r
- case MVT::i8: MovInst = X86::MOVZX32rr8; break;\r
- case MVT::i16: MovInst = X86::MOVZX32rr16; break;\r
- case MVT::i32: MovInst = X86::MOV32rr; break;\r
- default: llvm_unreachable("Unexpected zext to i64 source type");\r
- }\r
-\r
- unsigned Result32 = createResultReg(&X86::GR32RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(MovInst), Result32)\r
- .addReg(ResultReg);\r
-\r
- ResultReg = createResultReg(&X86::GR64RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::SUBREG_TO_REG),\r
- ResultReg)\r
- .addImm(0).addReg(Result32).addImm(X86::sub_32bit);\r
- } else if (DstVT != MVT::i8) {\r
- ResultReg = fastEmit_r(MVT::i8, DstVT.getSimpleVT(), ISD::ZERO_EXTEND,\r
- ResultReg, /*Kill=*/true);\r
- if (ResultReg == 0)\r
- return false;\r
- }\r
-\r
- updateValueMap(I, ResultReg);\r
- return true;\r
-}\r
-\r
-bool X86FastISel::X86SelectBranch(const Instruction *I) {\r
- // Unconditional branches are selected by tablegen-generated code.\r
- // Handle a conditional branch.\r
- const BranchInst *BI = cast<BranchInst>(I);\r
- MachineBasicBlock *TrueMBB = FuncInfo.MBBMap[BI->getSuccessor(0)];\r
- MachineBasicBlock *FalseMBB = FuncInfo.MBBMap[BI->getSuccessor(1)];\r
-\r
- // Fold the common case of a conditional branch with a comparison\r
- // in the same block (values defined on other blocks may not have\r
- // initialized registers).\r
- X86::CondCode CC;\r
- if (const CmpInst *CI = dyn_cast<CmpInst>(BI->getCondition())) {\r
- if (CI->hasOneUse() && CI->getParent() == I->getParent()) {\r
- EVT VT = TLI.getValueType(CI->getOperand(0)->getType());\r
-\r
- // Try to optimize or fold the cmp.\r
- CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);\r
- switch (Predicate) {\r
- default: break;\r
- case CmpInst::FCMP_FALSE: fastEmitBranch(FalseMBB, DbgLoc); return true;\r
- case CmpInst::FCMP_TRUE: fastEmitBranch(TrueMBB, DbgLoc); return true;\r
- }\r
-\r
- const Value *CmpLHS = CI->getOperand(0);\r
- const Value *CmpRHS = CI->getOperand(1);\r
-\r
- // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x,\r
- // 0.0.\r
- // We don't have to materialize a zero constant for this case and can just\r
- // use %x again on the RHS.\r
- if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {\r
- const auto *CmpRHSC = dyn_cast<ConstantFP>(CmpRHS);\r
- if (CmpRHSC && CmpRHSC->isNullValue())\r
- CmpRHS = CmpLHS;\r
- }\r
-\r
- // Try to take advantage of fallthrough opportunities.\r
- if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {\r
- std::swap(TrueMBB, FalseMBB);\r
- Predicate = CmpInst::getInversePredicate(Predicate);\r
- }\r
-\r
- // FCMP_OEQ and FCMP_UNE cannot be expressed with a single flag/condition\r
- // code check. Instead two branch instructions are required to check all\r
- // the flags. First we change the predicate to a supported condition code,\r
- // which will be the first branch. Later one we will emit the second\r
- // branch.\r
- bool NeedExtraBranch = false;\r
- switch (Predicate) {\r
- default: break;\r
- case CmpInst::FCMP_OEQ:\r
- std::swap(TrueMBB, FalseMBB); // fall-through\r
- case CmpInst::FCMP_UNE:\r
- NeedExtraBranch = true;\r
- Predicate = CmpInst::FCMP_ONE;\r
- break;\r
- }\r
-\r
- bool SwapArgs;\r
- unsigned BranchOpc;\r
- std::tie(CC, SwapArgs) = getX86ConditionCode(Predicate);\r
- assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");\r
-\r
- BranchOpc = X86::GetCondBranchFromCond(CC);\r
- if (SwapArgs)\r
- std::swap(CmpLHS, CmpRHS);\r
-\r
- // Emit a compare of the LHS and RHS, setting the flags.\r
- if (!X86FastEmitCompare(CmpLHS, CmpRHS, VT, CI->getDebugLoc()))\r
- return false;\r
-\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(BranchOpc))\r
- .addMBB(TrueMBB);\r
-\r
- // X86 requires a second branch to handle UNE (and OEQ, which is mapped\r
- // to UNE above).\r
- if (NeedExtraBranch) {\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::JP_1))\r
- .addMBB(TrueMBB);\r
- }\r
-\r
- // Obtain the branch weight and add the TrueBB to the successor list.\r
- uint32_t BranchWeight = 0;\r
- if (FuncInfo.BPI)\r
- BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->getParent(),\r
- TrueMBB->getBasicBlock());\r
- FuncInfo.MBB->addSuccessor(TrueMBB, BranchWeight);\r
-\r
- // Emits an unconditional branch to the FalseBB, obtains the branch\r
- // weight, and adds it to the successor list.\r
- fastEmitBranch(FalseMBB, DbgLoc);\r
-\r
- return true;\r
- }\r
- } else if (TruncInst *TI = dyn_cast<TruncInst>(BI->getCondition())) {\r
- // Handle things like "%cond = trunc i32 %X to i1 / br i1 %cond", which\r
- // typically happen for _Bool and C++ bools.\r
- MVT SourceVT;\r
- if (TI->hasOneUse() && TI->getParent() == I->getParent() &&\r
- isTypeLegal(TI->getOperand(0)->getType(), SourceVT)) {\r
- unsigned TestOpc = 0;\r
- switch (SourceVT.SimpleTy) {\r
- default: break;\r
- case MVT::i8: TestOpc = X86::TEST8ri; break;\r
- case MVT::i16: TestOpc = X86::TEST16ri; break;\r
- case MVT::i32: TestOpc = X86::TEST32ri; break;\r
- case MVT::i64: TestOpc = X86::TEST64ri32; break;\r
- }\r
- if (TestOpc) {\r
- unsigned OpReg = getRegForValue(TI->getOperand(0));\r
- if (OpReg == 0) return false;\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TestOpc))\r
- .addReg(OpReg).addImm(1);\r
-\r
- unsigned JmpOpc = X86::JNE_1;\r
- if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {\r
- std::swap(TrueMBB, FalseMBB);\r
- JmpOpc = X86::JE_1;\r
- }\r
-\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(JmpOpc))\r
- .addMBB(TrueMBB);\r
- fastEmitBranch(FalseMBB, DbgLoc);\r
- uint32_t BranchWeight = 0;\r
- if (FuncInfo.BPI)\r
- BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->getParent(),\r
- TrueMBB->getBasicBlock());\r
- FuncInfo.MBB->addSuccessor(TrueMBB, BranchWeight);\r
- return true;\r
- }\r
- }\r
- } else if (foldX86XALUIntrinsic(CC, BI, BI->getCondition())) {\r
- // Fake request the condition, otherwise the intrinsic might be completely\r
- // optimized away.\r
- unsigned TmpReg = getRegForValue(BI->getCondition());\r
- if (TmpReg == 0)\r
- return false;\r
-\r
- unsigned BranchOpc = X86::GetCondBranchFromCond(CC);\r
-\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(BranchOpc))\r
- .addMBB(TrueMBB);\r
- fastEmitBranch(FalseMBB, DbgLoc);\r
- uint32_t BranchWeight = 0;\r
- if (FuncInfo.BPI)\r
- BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->getParent(),\r
- TrueMBB->getBasicBlock());\r
- FuncInfo.MBB->addSuccessor(TrueMBB, BranchWeight);\r
- return true;\r
- }\r
-\r
- // Otherwise do a clumsy setcc and re-test it.\r
- // Note that i1 essentially gets ANY_EXTEND'ed to i8 where it isn't used\r
- // in an explicit cast, so make sure to handle that correctly.\r
- unsigned OpReg = getRegForValue(BI->getCondition());\r
- if (OpReg == 0) return false;\r
-\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TEST8ri))\r
- .addReg(OpReg).addImm(1);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::JNE_1))\r
- .addMBB(TrueMBB);\r
- fastEmitBranch(FalseMBB, DbgLoc);\r
- uint32_t BranchWeight = 0;\r
- if (FuncInfo.BPI)\r
- BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->getParent(),\r
- TrueMBB->getBasicBlock());\r
- FuncInfo.MBB->addSuccessor(TrueMBB, BranchWeight);\r
- return true;\r
-}\r
-\r
-bool X86FastISel::X86SelectShift(const Instruction *I) {\r
- unsigned CReg = 0, OpReg = 0;\r
- const TargetRegisterClass *RC = nullptr;\r
- if (I->getType()->isIntegerTy(8)) {\r
- CReg = X86::CL;\r
- RC = &X86::GR8RegClass;\r
- switch (I->getOpcode()) {\r
- case Instruction::LShr: OpReg = X86::SHR8rCL; break;\r
- case Instruction::AShr: OpReg = X86::SAR8rCL; break;\r
- case Instruction::Shl: OpReg = X86::SHL8rCL; break;\r
- default: return false;\r
- }\r
- } else if (I->getType()->isIntegerTy(16)) {\r
- CReg = X86::CX;\r
- RC = &X86::GR16RegClass;\r
- switch (I->getOpcode()) {\r
- case Instruction::LShr: OpReg = X86::SHR16rCL; break;\r
- case Instruction::AShr: OpReg = X86::SAR16rCL; break;\r
- case Instruction::Shl: OpReg = X86::SHL16rCL; break;\r
- default: return false;\r
- }\r
- } else if (I->getType()->isIntegerTy(32)) {\r
- CReg = X86::ECX;\r
- RC = &X86::GR32RegClass;\r
- switch (I->getOpcode()) {\r
- case Instruction::LShr: OpReg = X86::SHR32rCL; break;\r
- case Instruction::AShr: OpReg = X86::SAR32rCL; break;\r
- case Instruction::Shl: OpReg = X86::SHL32rCL; break;\r
- default: return false;\r
- }\r
- } else if (I->getType()->isIntegerTy(64)) {\r
- CReg = X86::RCX;\r
- RC = &X86::GR64RegClass;\r
- switch (I->getOpcode()) {\r
- case Instruction::LShr: OpReg = X86::SHR64rCL; break;\r
- case Instruction::AShr: OpReg = X86::SAR64rCL; break;\r
- case Instruction::Shl: OpReg = X86::SHL64rCL; break;\r
- default: return false;\r
- }\r
- } else {\r
- return false;\r
- }\r
-\r
- MVT VT;\r
- if (!isTypeLegal(I->getType(), VT))\r
- return false;\r
-\r
- unsigned Op0Reg = getRegForValue(I->getOperand(0));\r
- if (Op0Reg == 0) return false;\r
-\r
- unsigned Op1Reg = getRegForValue(I->getOperand(1));\r
- if (Op1Reg == 0) return false;\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),\r
- CReg).addReg(Op1Reg);\r
-\r
- // The shift instruction uses X86::CL. If we defined a super-register\r
- // of X86::CL, emit a subreg KILL to precisely describe what we're doing here.\r
- if (CReg != X86::CL)\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::KILL), X86::CL)\r
- .addReg(CReg, RegState::Kill);\r
-\r
- unsigned ResultReg = createResultReg(RC);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(OpReg), ResultReg)\r
- .addReg(Op0Reg);\r
- updateValueMap(I, ResultReg);\r
- return true;\r
-}\r
-\r
-bool X86FastISel::X86SelectDivRem(const Instruction *I) {\r
- const static unsigned NumTypes = 4; // i8, i16, i32, i64\r
- const static unsigned NumOps = 4; // SDiv, SRem, UDiv, URem\r
- const static bool S = true; // IsSigned\r
- const static bool U = false; // !IsSigned\r
- const static unsigned Copy = TargetOpcode::COPY;\r
- // For the X86 DIV/IDIV instruction, in most cases the dividend\r
- // (numerator) must be in a specific register pair highreg:lowreg,\r
- // producing the quotient in lowreg and the remainder in highreg.\r
- // For most data types, to set up the instruction, the dividend is\r
- // copied into lowreg, and lowreg is sign-extended or zero-extended\r
- // into highreg. The exception is i8, where the dividend is defined\r
- // as a single register rather than a register pair, and we\r
- // therefore directly sign-extend or zero-extend the dividend into\r
- // lowreg, instead of copying, and ignore the highreg.\r
- const static struct DivRemEntry {\r
- // The following portion depends only on the data type.\r
- const TargetRegisterClass *RC;\r
- unsigned LowInReg; // low part of the register pair\r
- unsigned HighInReg; // high part of the register pair\r
- // The following portion depends on both the data type and the operation.\r
- struct DivRemResult {\r
- unsigned OpDivRem; // The specific DIV/IDIV opcode to use.\r
- unsigned OpSignExtend; // Opcode for sign-extending lowreg into\r
- // highreg, or copying a zero into highreg.\r
- unsigned OpCopy; // Opcode for copying dividend into lowreg, or\r
- // zero/sign-extending into lowreg for i8.\r
- unsigned DivRemResultReg; // Register containing the desired result.\r
- bool IsOpSigned; // Whether to use signed or unsigned form.\r
- } ResultTable[NumOps];\r
- } OpTable[NumTypes] = {\r
- { &X86::GR8RegClass, X86::AX, 0, {\r
- { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S }, // SDiv\r
- { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S }, // SRem\r
- { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U }, // UDiv\r
- { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U }, // URem\r
- }\r
- }, // i8\r
- { &X86::GR16RegClass, X86::AX, X86::DX, {\r
- { X86::IDIV16r, X86::CWD, Copy, X86::AX, S }, // SDiv\r
- { X86::IDIV16r, X86::CWD, Copy, X86::DX, S }, // SRem\r
- { X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U }, // UDiv\r
- { X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U }, // URem\r
- }\r
- }, // i16\r
- { &X86::GR32RegClass, X86::EAX, X86::EDX, {\r
- { X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S }, // SDiv\r
- { X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S }, // SRem\r
- { X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U }, // UDiv\r
- { X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U }, // URem\r
- }\r
- }, // i32\r
- { &X86::GR64RegClass, X86::RAX, X86::RDX, {\r
- { X86::IDIV64r, X86::CQO, Copy, X86::RAX, S }, // SDiv\r
- { X86::IDIV64r, X86::CQO, Copy, X86::RDX, S }, // SRem\r
- { X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U }, // UDiv\r
- { X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U }, // URem\r
- }\r
- }, // i64\r
- };\r
-\r
- MVT VT;\r
- if (!isTypeLegal(I->getType(), VT))\r
- return false;\r
-\r
- unsigned TypeIndex, OpIndex;\r
- switch (VT.SimpleTy) {\r
- default: return false;\r
- case MVT::i8: TypeIndex = 0; break;\r
- case MVT::i16: TypeIndex = 1; break;\r
- case MVT::i32: TypeIndex = 2; break;\r
- case MVT::i64: TypeIndex = 3;\r
- if (!Subtarget->is64Bit())\r
- return false;\r
- break;\r
- }\r
-\r
- switch (I->getOpcode()) {\r
- default: llvm_unreachable("Unexpected div/rem opcode");\r
- case Instruction::SDiv: OpIndex = 0; break;\r
- case Instruction::SRem: OpIndex = 1; break;\r
- case Instruction::UDiv: OpIndex = 2; break;\r
- case Instruction::URem: OpIndex = 3; break;\r
- }\r
-\r
- const DivRemEntry &TypeEntry = OpTable[TypeIndex];\r
- const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];\r
- unsigned Op0Reg = getRegForValue(I->getOperand(0));\r
- if (Op0Reg == 0)\r
- return false;\r
- unsigned Op1Reg = getRegForValue(I->getOperand(1));\r
- if (Op1Reg == 0)\r
- return false;\r
-\r
- // Move op0 into low-order input register.\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(OpEntry.OpCopy), TypeEntry.LowInReg).addReg(Op0Reg);\r
- // Zero-extend or sign-extend into high-order input register.\r
- if (OpEntry.OpSignExtend) {\r
- if (OpEntry.IsOpSigned)\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(OpEntry.OpSignExtend));\r
- else {\r
- unsigned Zero32 = createResultReg(&X86::GR32RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(X86::MOV32r0), Zero32);\r
-\r
- // Copy the zero into the appropriate sub/super/identical physical\r
- // register. Unfortunately the operations needed are not uniform enough\r
- // to fit neatly into the table above.\r
- if (VT.SimpleTy == MVT::i16) {\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Copy), TypeEntry.HighInReg)\r
- .addReg(Zero32, 0, X86::sub_16bit);\r
- } else if (VT.SimpleTy == MVT::i32) {\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Copy), TypeEntry.HighInReg)\r
- .addReg(Zero32);\r
- } else if (VT.SimpleTy == MVT::i64) {\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)\r
- .addImm(0).addReg(Zero32).addImm(X86::sub_32bit);\r
- }\r
- }\r
- }\r
- // Generate the DIV/IDIV instruction.\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(OpEntry.OpDivRem)).addReg(Op1Reg);\r
- // For i8 remainder, we can't reference AH directly, as we'll end\r
- // up with bogus copies like %R9B = COPY %AH. Reference AX\r
- // instead to prevent AH references in a REX instruction.\r
- //\r
- // The current assumption of the fast register allocator is that isel\r
- // won't generate explicit references to the GPR8_NOREX registers. If\r
- // the allocator and/or the backend get enhanced to be more robust in\r
- // that regard, this can be, and should be, removed.\r
- unsigned ResultReg = 0;\r
- if ((I->getOpcode() == Instruction::SRem ||\r
- I->getOpcode() == Instruction::URem) &&\r
- OpEntry.DivRemResultReg == X86::AH && Subtarget->is64Bit()) {\r
- unsigned SourceSuperReg = createResultReg(&X86::GR16RegClass);\r
- unsigned ResultSuperReg = createResultReg(&X86::GR16RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Copy), SourceSuperReg).addReg(X86::AX);\r
-\r
- // Shift AX right by 8 bits instead of using AH.\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::SHR16ri),\r
- ResultSuperReg).addReg(SourceSuperReg).addImm(8);\r
-\r
- // Now reference the 8-bit subreg of the result.\r
- ResultReg = fastEmitInst_extractsubreg(MVT::i8, ResultSuperReg,\r
- /*Kill=*/true, X86::sub_8bit);\r
- }\r
- // Copy the result out of the physreg if we haven't already.\r
- if (!ResultReg) {\r
- ResultReg = createResultReg(TypeEntry.RC);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Copy), ResultReg)\r
- .addReg(OpEntry.DivRemResultReg);\r
- }\r
- updateValueMap(I, ResultReg);\r
-\r
- return true;\r
-}\r
-\r
-/// \brief Emit a conditional move instruction (if the are supported) to lower\r
-/// the select.\r
-bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I) {\r
- // Check if the subtarget supports these instructions.\r
- if (!Subtarget->hasCMov())\r
- return false;\r
-\r
- // FIXME: Add support for i8.\r
- if (RetVT < MVT::i16 || RetVT > MVT::i64)\r
- return false;\r
-\r
- const Value *Cond = I->getOperand(0);\r
- const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);\r
- bool NeedTest = true;\r
- X86::CondCode CC = X86::COND_NE;\r
-\r
- // Optimize conditions coming from a compare if both instructions are in the\r
- // same basic block (values defined in other basic blocks may not have\r
- // initialized registers).\r
- const auto *CI = dyn_cast<CmpInst>(Cond);\r
- if (CI && (CI->getParent() == I->getParent())) {\r
- CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);\r
-\r
- // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.\r
- static unsigned SETFOpcTable[2][3] = {\r
- { X86::SETNPr, X86::SETEr , X86::TEST8rr },\r
- { X86::SETPr, X86::SETNEr, X86::OR8rr }\r
- };\r
- unsigned *SETFOpc = nullptr;\r
- switch (Predicate) {\r
- default: break;\r
- case CmpInst::FCMP_OEQ:\r
- SETFOpc = &SETFOpcTable[0][0];\r
- Predicate = CmpInst::ICMP_NE;\r
- break;\r
- case CmpInst::FCMP_UNE:\r
- SETFOpc = &SETFOpcTable[1][0];\r
- Predicate = CmpInst::ICMP_NE;\r
- break;\r
- }\r
-\r
- bool NeedSwap;\r
- std::tie(CC, NeedSwap) = getX86ConditionCode(Predicate);\r
- assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");\r
-\r
- const Value *CmpLHS = CI->getOperand(0);\r
- const Value *CmpRHS = CI->getOperand(1);\r
- if (NeedSwap)\r
- std::swap(CmpLHS, CmpRHS);\r
-\r
- EVT CmpVT = TLI.getValueType(CmpLHS->getType());\r
- // Emit a compare of the LHS and RHS, setting the flags.\r
- if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))\r
- return false;\r
-\r
- if (SETFOpc) {\r
- unsigned FlagReg1 = createResultReg(&X86::GR8RegClass);\r
- unsigned FlagReg2 = createResultReg(&X86::GR8RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[0]),\r
- FlagReg1);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[1]),\r
- FlagReg2);\r
- auto const &II = TII.get(SETFOpc[2]);\r
- if (II.getNumDefs()) {\r
- unsigned TmpReg = createResultReg(&X86::GR8RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, TmpReg)\r
- .addReg(FlagReg2).addReg(FlagReg1);\r
- } else {\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)\r
- .addReg(FlagReg2).addReg(FlagReg1);\r
- }\r
- }\r
- NeedTest = false;\r
- } else if (foldX86XALUIntrinsic(CC, I, Cond)) {\r
- // Fake request the condition, otherwise the intrinsic might be completely\r
- // optimized away.\r
- unsigned TmpReg = getRegForValue(Cond);\r
- if (TmpReg == 0)\r
- return false;\r
-\r
- NeedTest = false;\r
- }\r
-\r
- if (NeedTest) {\r
- // Selects operate on i1, however, CondReg is 8 bits width and may contain\r
- // garbage. Indeed, only the less significant bit is supposed to be\r
- // accurate. If we read more than the lsb, we may see non-zero values\r
- // whereas lsb is zero. Therefore, we have to truncate Op0Reg to i1 for\r
- // the select. This is achieved by performing TEST against 1.\r
- unsigned CondReg = getRegForValue(Cond);\r
- if (CondReg == 0)\r
- return false;\r
- bool CondIsKill = hasTrivialKill(Cond);\r
-\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TEST8ri))\r
- .addReg(CondReg, getKillRegState(CondIsKill)).addImm(1);\r
- }\r
-\r
- const Value *LHS = I->getOperand(1);\r
- const Value *RHS = I->getOperand(2);\r
-\r
- unsigned RHSReg = getRegForValue(RHS);\r
- bool RHSIsKill = hasTrivialKill(RHS);\r
-\r
- unsigned LHSReg = getRegForValue(LHS);\r
- bool LHSIsKill = hasTrivialKill(LHS);\r
-\r
- if (!LHSReg || !RHSReg)\r
- return false;\r
-\r
- unsigned Opc = X86::getCMovFromCond(CC, RC->getSize());\r
- unsigned ResultReg = fastEmitInst_rr(Opc, RC, RHSReg, RHSIsKill,\r
- LHSReg, LHSIsKill);\r
- updateValueMap(I, ResultReg);\r
- return true;\r
-}\r
-\r
-/// \brief Emit SSE instructions to lower the select.\r
-///\r
-/// Try to use SSE1/SSE2 instructions to simulate a select without branches.\r
-/// This lowers fp selects into a CMP/AND/ANDN/OR sequence when the necessary\r
-/// SSE instructions are available.\r
-bool X86FastISel::X86FastEmitSSESelect(MVT RetVT, const Instruction *I) {\r
- // Optimize conditions coming from a compare if both instructions are in the\r
- // same basic block (values defined in other basic blocks may not have\r
- // initialized registers).\r
- const auto *CI = dyn_cast<FCmpInst>(I->getOperand(0));\r
- if (!CI || (CI->getParent() != I->getParent()))\r
- return false;\r
-\r
- if (I->getType() != CI->getOperand(0)->getType() ||\r
- !((Subtarget->hasSSE1() && RetVT == MVT::f32) ||\r
- (Subtarget->hasSSE2() && RetVT == MVT::f64)))\r
- return false;\r
-\r
- const Value *CmpLHS = CI->getOperand(0);\r
- const Value *CmpRHS = CI->getOperand(1);\r
- CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);\r
-\r
- // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.\r
- // We don't have to materialize a zero constant for this case and can just use\r
- // %x again on the RHS.\r
- if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {\r
- const auto *CmpRHSC = dyn_cast<ConstantFP>(CmpRHS);\r
- if (CmpRHSC && CmpRHSC->isNullValue())\r
- CmpRHS = CmpLHS;\r
- }\r
-\r
- unsigned CC;\r
- bool NeedSwap;\r
- std::tie(CC, NeedSwap) = getX86SSEConditionCode(Predicate);\r
- if (CC > 7)\r
- return false;\r
-\r
- if (NeedSwap)\r
- std::swap(CmpLHS, CmpRHS);\r
-\r
- static unsigned OpcTable[2][2][4] = {\r
- { { X86::CMPSSrr, X86::FsANDPSrr, X86::FsANDNPSrr, X86::FsORPSrr },\r
- { X86::VCMPSSrr, X86::VFsANDPSrr, X86::VFsANDNPSrr, X86::VFsORPSrr } },\r
- { { X86::CMPSDrr, X86::FsANDPDrr, X86::FsANDNPDrr, X86::FsORPDrr },\r
- { X86::VCMPSDrr, X86::VFsANDPDrr, X86::VFsANDNPDrr, X86::VFsORPDrr } }\r
- };\r
-\r
- bool HasAVX = Subtarget->hasAVX();\r
- unsigned *Opc = nullptr;\r
- switch (RetVT.SimpleTy) {\r
- default: return false;\r
- case MVT::f32: Opc = &OpcTable[0][HasAVX][0]; break;\r
- case MVT::f64: Opc = &OpcTable[1][HasAVX][0]; break;\r
- }\r
-\r
- const Value *LHS = I->getOperand(1);\r
- const Value *RHS = I->getOperand(2);\r
-\r
- unsigned LHSReg = getRegForValue(LHS);\r
- bool LHSIsKill = hasTrivialKill(LHS);\r
-\r
- unsigned RHSReg = getRegForValue(RHS);\r
- bool RHSIsKill = hasTrivialKill(RHS);\r
-\r
- unsigned CmpLHSReg = getRegForValue(CmpLHS);\r
- bool CmpLHSIsKill = hasTrivialKill(CmpLHS);\r
-\r
- unsigned CmpRHSReg = getRegForValue(CmpRHS);\r
- bool CmpRHSIsKill = hasTrivialKill(CmpRHS);\r
-\r
- if (!LHSReg || !RHSReg || !CmpLHS || !CmpRHS)\r
- return false;\r
-\r
- const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);\r
- unsigned CmpReg = fastEmitInst_rri(Opc[0], RC, CmpLHSReg, CmpLHSIsKill,\r
- CmpRHSReg, CmpRHSIsKill, CC);\r
- unsigned AndReg = fastEmitInst_rr(Opc[1], RC, CmpReg, /*IsKill=*/false,\r
- LHSReg, LHSIsKill);\r
- unsigned AndNReg = fastEmitInst_rr(Opc[2], RC, CmpReg, /*IsKill=*/true,\r
- RHSReg, RHSIsKill);\r
- unsigned ResultReg = fastEmitInst_rr(Opc[3], RC, AndNReg, /*IsKill=*/true,\r
- AndReg, /*IsKill=*/true);\r
- updateValueMap(I, ResultReg);\r
- return true;\r
-}\r
-\r
-bool X86FastISel::X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I) {\r
- // These are pseudo CMOV instructions and will be later expanded into control-\r
- // flow.\r
- unsigned Opc;\r
- switch (RetVT.SimpleTy) {\r
- default: return false;\r
- case MVT::i8: Opc = X86::CMOV_GR8; break;\r
- case MVT::i16: Opc = X86::CMOV_GR16; break;\r
- case MVT::i32: Opc = X86::CMOV_GR32; break;\r
- case MVT::f32: Opc = X86::CMOV_FR32; break;\r
- case MVT::f64: Opc = X86::CMOV_FR64; break;\r
- }\r
-\r
- const Value *Cond = I->getOperand(0);\r
- X86::CondCode CC = X86::COND_NE;\r
-\r
- // Optimize conditions coming from a compare if both instructions are in the\r
- // same basic block (values defined in other basic blocks may not have\r
- // initialized registers).\r
- const auto *CI = dyn_cast<CmpInst>(Cond);\r
- if (CI && (CI->getParent() == I->getParent())) {\r
- bool NeedSwap;\r
- std::tie(CC, NeedSwap) = getX86ConditionCode(CI->getPredicate());\r
- if (CC > X86::LAST_VALID_COND)\r
- return false;\r
-\r
- const Value *CmpLHS = CI->getOperand(0);\r
- const Value *CmpRHS = CI->getOperand(1);\r
-\r
- if (NeedSwap)\r
- std::swap(CmpLHS, CmpRHS);\r
-\r
- EVT CmpVT = TLI.getValueType(CmpLHS->getType());\r
- if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))\r
- return false;\r
- } else {\r
- unsigned CondReg = getRegForValue(Cond);\r
- if (CondReg == 0)\r
- return false;\r
- bool CondIsKill = hasTrivialKill(Cond);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TEST8ri))\r
- .addReg(CondReg, getKillRegState(CondIsKill)).addImm(1);\r
- }\r
-\r
- const Value *LHS = I->getOperand(1);\r
- const Value *RHS = I->getOperand(2);\r
-\r
- unsigned LHSReg = getRegForValue(LHS);\r
- bool LHSIsKill = hasTrivialKill(LHS);\r
-\r
- unsigned RHSReg = getRegForValue(RHS);\r
- bool RHSIsKill = hasTrivialKill(RHS);\r
-\r
- if (!LHSReg || !RHSReg)\r
- return false;\r
-\r
- const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);\r
-\r
- unsigned ResultReg =\r
- fastEmitInst_rri(Opc, RC, RHSReg, RHSIsKill, LHSReg, LHSIsKill, CC);\r
- updateValueMap(I, ResultReg);\r
- return true;\r
-}\r
-\r
-bool X86FastISel::X86SelectSelect(const Instruction *I) {\r
- MVT RetVT;\r
- if (!isTypeLegal(I->getType(), RetVT))\r
- return false;\r
-\r
- // Check if we can fold the select.\r
- if (const auto *CI = dyn_cast<CmpInst>(I->getOperand(0))) {\r
- CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);\r
- const Value *Opnd = nullptr;\r
- switch (Predicate) {\r
- default: break;\r
- case CmpInst::FCMP_FALSE: Opnd = I->getOperand(2); break;\r
- case CmpInst::FCMP_TRUE: Opnd = I->getOperand(1); break;\r
- }\r
- // No need for a select anymore - this is an unconditional move.\r
- if (Opnd) {\r
- unsigned OpReg = getRegForValue(Opnd);\r
- if (OpReg == 0)\r
- return false;\r
- bool OpIsKill = hasTrivialKill(Opnd);\r
- const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);\r
- unsigned ResultReg = createResultReg(RC);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::COPY), ResultReg)\r
- .addReg(OpReg, getKillRegState(OpIsKill));\r
- updateValueMap(I, ResultReg);\r
- return true;\r
- }\r
- }\r
-\r
- // First try to use real conditional move instructions.\r
- if (X86FastEmitCMoveSelect(RetVT, I))\r
- return true;\r
-\r
- // Try to use a sequence of SSE instructions to simulate a conditional move.\r
- if (X86FastEmitSSESelect(RetVT, I))\r
- return true;\r
-\r
- // Fall-back to pseudo conditional move instructions, which will be later\r
- // converted to control-flow.\r
- if (X86FastEmitPseudoSelect(RetVT, I))\r
- return true;\r
-\r
- return false;\r
-}\r
-\r
-bool X86FastISel::X86SelectFPExt(const Instruction *I) {\r
- // fpext from float to double.\r
- if (X86ScalarSSEf64 &&\r
- I->getType()->isDoubleTy()) {\r
- const Value *V = I->getOperand(0);\r
- if (V->getType()->isFloatTy()) {\r
- unsigned OpReg = getRegForValue(V);\r
- if (OpReg == 0) return false;\r
- unsigned ResultReg = createResultReg(&X86::FR64RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(X86::CVTSS2SDrr), ResultReg)\r
- .addReg(OpReg);\r
- updateValueMap(I, ResultReg);\r
- return true;\r
- }\r
- }\r
-\r
- return false;\r
-}\r
-\r
-bool X86FastISel::X86SelectFPTrunc(const Instruction *I) {\r
- if (X86ScalarSSEf64) {\r
- if (I->getType()->isFloatTy()) {\r
- const Value *V = I->getOperand(0);\r
- if (V->getType()->isDoubleTy()) {\r
- unsigned OpReg = getRegForValue(V);\r
- if (OpReg == 0) return false;\r
- unsigned ResultReg = createResultReg(&X86::FR32RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(X86::CVTSD2SSrr), ResultReg)\r
- .addReg(OpReg);\r
- updateValueMap(I, ResultReg);\r
- return true;\r
- }\r
- }\r
- }\r
-\r
- return false;\r
-}\r
-\r
-bool X86FastISel::X86SelectTrunc(const Instruction *I) {\r
- EVT SrcVT = TLI.getValueType(I->getOperand(0)->getType());\r
- EVT DstVT = TLI.getValueType(I->getType());\r
-\r
- // This code only handles truncation to byte.\r
- if (DstVT != MVT::i8 && DstVT != MVT::i1)\r
- return false;\r
- if (!TLI.isTypeLegal(SrcVT))\r
- return false;\r
-\r
- unsigned InputReg = getRegForValue(I->getOperand(0));\r
- if (!InputReg)\r
- // Unhandled operand. Halt "fast" selection and bail.\r
- return false;\r
-\r
- if (SrcVT == MVT::i8) {\r
- // Truncate from i8 to i1; no code needed.\r
- updateValueMap(I, InputReg);\r
- return true;\r
- }\r
-\r
- if (!Subtarget->is64Bit()) {\r
- // If we're on x86-32; we can't extract an i8 from a general register.\r
- // First issue a copy to GR16_ABCD or GR32_ABCD.\r
- const TargetRegisterClass *CopyRC =\r
- (SrcVT == MVT::i16) ? &X86::GR16_ABCDRegClass : &X86::GR32_ABCDRegClass;\r
- unsigned CopyReg = createResultReg(CopyRC);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::COPY), CopyReg).addReg(InputReg);\r
- InputReg = CopyReg;\r
- }\r
-\r
- // Issue an extract_subreg.\r
- unsigned ResultReg = fastEmitInst_extractsubreg(MVT::i8,\r
- InputReg, /*Kill=*/true,\r
- X86::sub_8bit);\r
- if (!ResultReg)\r
- return false;\r
-\r
- updateValueMap(I, ResultReg);\r
- return true;\r
-}\r
-\r
-bool X86FastISel::IsMemcpySmall(uint64_t Len) {\r
- return Len <= (Subtarget->is64Bit() ? 32 : 16);\r
-}\r
-\r
-bool X86FastISel::TryEmitSmallMemcpy(X86AddressMode DestAM,\r
- X86AddressMode SrcAM, uint64_t Len) {\r
-\r
- // Make sure we don't bloat code by inlining very large memcpy's.\r
- if (!IsMemcpySmall(Len))\r
- return false;\r
-\r
- bool i64Legal = Subtarget->is64Bit();\r
-\r
- // We don't care about alignment here since we just emit integer accesses.\r
- while (Len) {\r
- MVT VT;\r
- if (Len >= 8 && i64Legal)\r
- VT = MVT::i64;\r
- else if (Len >= 4)\r
- VT = MVT::i32;\r
- else if (Len >= 2)\r
- VT = MVT::i16;\r
- else\r
- VT = MVT::i8;\r
-\r
- unsigned Reg;\r
- bool RV = X86FastEmitLoad(VT, SrcAM, nullptr, Reg);\r
- RV &= X86FastEmitStore(VT, Reg, /*Kill=*/true, DestAM);\r
- assert(RV && "Failed to emit load or store??");\r
-\r
- unsigned Size = VT.getSizeInBits()/8;\r
- Len -= Size;\r
- DestAM.Disp += Size;\r
- SrcAM.Disp += Size;\r
- }\r
-\r
- return true;\r
-}\r
-\r
-bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {\r
- // FIXME: Handle more intrinsics.\r
- switch (II->getIntrinsicID()) {\r
- default: return false;\r
- case Intrinsic::frameaddress: {\r
- Type *RetTy = II->getCalledFunction()->getReturnType();\r
-\r
- MVT VT;\r
- if (!isTypeLegal(RetTy, VT))\r
- return false;\r
-\r
- unsigned Opc;\r
- const TargetRegisterClass *RC = nullptr;\r
-\r
- switch (VT.SimpleTy) {\r
- default: llvm_unreachable("Invalid result type for frameaddress.");\r
- case MVT::i32: Opc = X86::MOV32rm; RC = &X86::GR32RegClass; break;\r
- case MVT::i64: Opc = X86::MOV64rm; RC = &X86::GR64RegClass; break;\r
- }\r
-\r
- // This needs to be set before we call getPtrSizedFrameRegister, otherwise\r
- // we get the wrong frame register.\r
- MachineFrameInfo *MFI = FuncInfo.MF->getFrameInfo();\r
- MFI->setFrameAddressIsTaken(true);\r
-\r
- const X86RegisterInfo *RegInfo = static_cast<const X86RegisterInfo *>(\r
- TM.getSubtargetImpl()->getRegisterInfo());\r
- unsigned FrameReg = RegInfo->getPtrSizedFrameRegister(*(FuncInfo.MF));\r
- assert(((FrameReg == X86::RBP && VT == MVT::i64) ||\r
- (FrameReg == X86::EBP && VT == MVT::i32)) &&\r
- "Invalid Frame Register!");\r
-\r
- // Always make a copy of the frame register to to a vreg first, so that we\r
- // never directly reference the frame register (the TwoAddressInstruction-\r
- // Pass doesn't like that).\r
- unsigned SrcReg = createResultReg(RC);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::COPY), SrcReg).addReg(FrameReg);\r
-\r
- // Now recursively load from the frame address.\r
- // movq (%rbp), %rax\r
- // movq (%rax), %rax\r
- // movq (%rax), %rax\r
- // ...\r
- unsigned DestReg;\r
- unsigned Depth = cast<ConstantInt>(II->getOperand(0))->getZExtValue();\r
- while (Depth--) {\r
- DestReg = createResultReg(RC);\r
- addDirectMem(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Opc), DestReg), SrcReg);\r
- SrcReg = DestReg;\r
- }\r
-\r
- updateValueMap(II, SrcReg);\r
- return true;\r
- }\r
- case Intrinsic::memcpy: {\r
- const MemCpyInst *MCI = cast<MemCpyInst>(II);\r
- // Don't handle volatile or variable length memcpys.\r
- if (MCI->isVolatile())\r
- return false;\r
-\r
- if (isa<ConstantInt>(MCI->getLength())) {\r
- // Small memcpy's are common enough that we want to do them\r
- // without a call if possible.\r
- uint64_t Len = cast<ConstantInt>(MCI->getLength())->getZExtValue();\r
- if (IsMemcpySmall(Len)) {\r
- X86AddressMode DestAM, SrcAM;\r
- if (!X86SelectAddress(MCI->getRawDest(), DestAM) ||\r
- !X86SelectAddress(MCI->getRawSource(), SrcAM))\r
- return false;\r
- TryEmitSmallMemcpy(DestAM, SrcAM, Len);\r
- return true;\r
- }\r
- }\r
-\r
- unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;\r
- if (!MCI->getLength()->getType()->isIntegerTy(SizeWidth))\r
- return false;\r
-\r
- if (MCI->getSourceAddressSpace() > 255 || MCI->getDestAddressSpace() > 255)\r
- return false;\r
-\r
- return lowerCallTo(II, "memcpy", II->getNumArgOperands() - 2);\r
- }\r
- case Intrinsic::memset: {\r
- const MemSetInst *MSI = cast<MemSetInst>(II);\r
-\r
- if (MSI->isVolatile())\r
- return false;\r
-\r
- unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;\r
- if (!MSI->getLength()->getType()->isIntegerTy(SizeWidth))\r
- return false;\r
-\r
- if (MSI->getDestAddressSpace() > 255)\r
- return false;\r
-\r
- return lowerCallTo(II, "memset", II->getNumArgOperands() - 2);\r
- }\r
- case Intrinsic::stackprotector: {\r
- // Emit code to store the stack guard onto the stack.\r
- EVT PtrTy = TLI.getPointerTy();\r
-\r
- const Value *Op1 = II->getArgOperand(0); // The guard's value.\r
- const AllocaInst *Slot = cast<AllocaInst>(II->getArgOperand(1));\r
-\r
- MFI.setStackProtectorIndex(FuncInfo.StaticAllocaMap[Slot]);\r
-\r
- // Grab the frame index.\r
- X86AddressMode AM;\r
- if (!X86SelectAddress(Slot, AM)) return false;\r
- if (!X86FastEmitStore(PtrTy, Op1, AM)) return false;\r
- return true;\r
- }\r
- case Intrinsic::dbg_declare: {\r
- const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);\r
- X86AddressMode AM;\r
- assert(DI->getAddress() && "Null address should be checked earlier!");\r
- if (!X86SelectAddress(DI->getAddress(), AM))\r
- return false;\r
- const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);\r
- // FIXME may need to add RegState::Debug to any registers produced,\r
- // although ESP/EBP should be the only ones at the moment.\r
- addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II), AM)\r
- .addImm(0)\r
- .addMetadata(DI->getVariable())\r
- .addMetadata(DI->getExpression());\r
- return true;\r
- }\r
- case Intrinsic::trap: {\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TRAP));\r
- return true;\r
- }\r
- case Intrinsic::sqrt: {\r
- if (!Subtarget->hasSSE1())\r
- return false;\r
-\r
- Type *RetTy = II->getCalledFunction()->getReturnType();\r
-\r
- MVT VT;\r
- if (!isTypeLegal(RetTy, VT))\r
- return false;\r
-\r
- // Unfortunately we can't use fastEmit_r, because the AVX version of FSQRT\r
- // is not generated by FastISel yet.\r
- // FIXME: Update this code once tablegen can handle it.\r
- static const unsigned SqrtOpc[2][2] = {\r
- {X86::SQRTSSr, X86::VSQRTSSr},\r
- {X86::SQRTSDr, X86::VSQRTSDr}\r
- };\r
- bool HasAVX = Subtarget->hasAVX();\r
- unsigned Opc;\r
- const TargetRegisterClass *RC;\r
- switch (VT.SimpleTy) {\r
- default: return false;\r
- case MVT::f32: Opc = SqrtOpc[0][HasAVX]; RC = &X86::FR32RegClass; break;\r
- case MVT::f64: Opc = SqrtOpc[1][HasAVX]; RC = &X86::FR64RegClass; break;\r
- }\r
-\r
- const Value *SrcVal = II->getArgOperand(0);\r
- unsigned SrcReg = getRegForValue(SrcVal);\r
-\r
- if (SrcReg == 0)\r
- return false;\r
-\r
- unsigned ImplicitDefReg = 0;\r
- if (HasAVX) {\r
- ImplicitDefReg = createResultReg(RC);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);\r
- }\r
-\r
- unsigned ResultReg = createResultReg(RC);\r
- MachineInstrBuilder MIB;\r
- MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc),\r
- ResultReg);\r
-\r
- if (ImplicitDefReg)\r
- MIB.addReg(ImplicitDefReg);\r
-\r
- MIB.addReg(SrcReg);\r
-\r
- updateValueMap(II, ResultReg);\r
- return true;\r
- }\r
- case Intrinsic::sadd_with_overflow:\r
- case Intrinsic::uadd_with_overflow:\r
- case Intrinsic::ssub_with_overflow:\r
- case Intrinsic::usub_with_overflow:\r
- case Intrinsic::smul_with_overflow:\r
- case Intrinsic::umul_with_overflow: {\r
- // This implements the basic lowering of the xalu with overflow intrinsics\r
- // into add/sub/mul followed by either seto or setb.\r
- const Function *Callee = II->getCalledFunction();\r
- auto *Ty = cast<StructType>(Callee->getReturnType());\r
- Type *RetTy = Ty->getTypeAtIndex(0U);\r
- Type *CondTy = Ty->getTypeAtIndex(1);\r
-\r
- MVT VT;\r
- if (!isTypeLegal(RetTy, VT))\r
- return false;\r
-\r
- if (VT < MVT::i8 || VT > MVT::i64)\r
- return false;\r
-\r
- const Value *LHS = II->getArgOperand(0);\r
- const Value *RHS = II->getArgOperand(1);\r
-\r
- // Canonicalize immediate to the RHS.\r
- if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) &&\r
- isCommutativeIntrinsic(II))\r
- std::swap(LHS, RHS);\r
-\r
- bool UseIncDec = false;\r
- if (isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isOne())\r
- UseIncDec = true;\r
-\r
- unsigned BaseOpc, CondOpc;\r
- switch (II->getIntrinsicID()) {\r
- default: llvm_unreachable("Unexpected intrinsic!");\r
- case Intrinsic::sadd_with_overflow:\r
- BaseOpc = UseIncDec ? unsigned(X86ISD::INC) : unsigned(ISD::ADD);\r
- CondOpc = X86::SETOr;\r
- break;\r
- case Intrinsic::uadd_with_overflow:\r
- BaseOpc = ISD::ADD; CondOpc = X86::SETBr; break;\r
- case Intrinsic::ssub_with_overflow:\r
- BaseOpc = UseIncDec ? unsigned(X86ISD::DEC) : unsigned(ISD::SUB);\r
- CondOpc = X86::SETOr;\r
- break;\r
- case Intrinsic::usub_with_overflow:\r
- BaseOpc = ISD::SUB; CondOpc = X86::SETBr; break;\r
- case Intrinsic::smul_with_overflow:\r
- BaseOpc = X86ISD::SMUL; CondOpc = X86::SETOr; break;\r
- case Intrinsic::umul_with_overflow:\r
- BaseOpc = X86ISD::UMUL; CondOpc = X86::SETOr; break;\r
- }\r
-\r
- unsigned LHSReg = getRegForValue(LHS);\r
- if (LHSReg == 0)\r
- return false;\r
- bool LHSIsKill = hasTrivialKill(LHS);\r
-\r
- unsigned ResultReg = 0;\r
- // Check if we have an immediate version.\r
- if (const auto *CI = dyn_cast<ConstantInt>(RHS)) {\r
- static const unsigned Opc[2][4] = {\r
- { X86::INC8r, X86::INC16r, X86::INC32r, X86::INC64r },\r
- { X86::DEC8r, X86::DEC16r, X86::DEC32r, X86::DEC64r }\r
- };\r
-\r
- if (BaseOpc == X86ISD::INC || BaseOpc == X86ISD::DEC) {\r
- ResultReg = createResultReg(TLI.getRegClassFor(VT));\r
- bool IsDec = BaseOpc == X86ISD::DEC;\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Opc[IsDec][VT.SimpleTy-MVT::i8]), ResultReg)\r
- .addReg(LHSReg, getKillRegState(LHSIsKill));\r
- } else\r
- ResultReg = fastEmit_ri(VT, VT, BaseOpc, LHSReg, LHSIsKill,\r
- CI->getZExtValue());\r
- }\r
-\r
- unsigned RHSReg;\r
- bool RHSIsKill;\r
- if (!ResultReg) {\r
- RHSReg = getRegForValue(RHS);\r
- if (RHSReg == 0)\r
- return false;\r
- RHSIsKill = hasTrivialKill(RHS);\r
- ResultReg = fastEmit_rr(VT, VT, BaseOpc, LHSReg, LHSIsKill, RHSReg,\r
- RHSIsKill);\r
- }\r
-\r
- // FastISel doesn't have a pattern for all X86::MUL*r and X86::IMUL*r. Emit\r
- // it manually.\r
- if (BaseOpc == X86ISD::UMUL && !ResultReg) {\r
- static const unsigned MULOpc[] =\r
- { X86::MUL8r, X86::MUL16r, X86::MUL32r, X86::MUL64r };\r
- static const unsigned Reg[] = { X86::AL, X86::AX, X86::EAX, X86::RAX };\r
- // First copy the first operand into RAX, which is an implicit input to\r
- // the X86::MUL*r instruction.\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::COPY), Reg[VT.SimpleTy-MVT::i8])\r
- .addReg(LHSReg, getKillRegState(LHSIsKill));\r
- ResultReg = fastEmitInst_r(MULOpc[VT.SimpleTy-MVT::i8],\r
- TLI.getRegClassFor(VT), RHSReg, RHSIsKill);\r
- } else if (BaseOpc == X86ISD::SMUL && !ResultReg) {\r
- static const unsigned MULOpc[] =\r
- { X86::IMUL8r, X86::IMUL16rr, X86::IMUL32rr, X86::IMUL64rr };\r
- if (VT == MVT::i8) {\r
- // Copy the first operand into AL, which is an implicit input to the\r
- // X86::IMUL8r instruction.\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::COPY), X86::AL)\r
- .addReg(LHSReg, getKillRegState(LHSIsKill));\r
- ResultReg = fastEmitInst_r(MULOpc[0], TLI.getRegClassFor(VT), RHSReg,\r
- RHSIsKill);\r
- } else\r
- ResultReg = fastEmitInst_rr(MULOpc[VT.SimpleTy-MVT::i8],\r
- TLI.getRegClassFor(VT), LHSReg, LHSIsKill,\r
- RHSReg, RHSIsKill);\r
- }\r
-\r
- if (!ResultReg)\r
- return false;\r
-\r
- unsigned ResultReg2 = FuncInfo.CreateRegs(CondTy);\r
- assert((ResultReg+1) == ResultReg2 && "Nonconsecutive result registers.");\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CondOpc),\r
- ResultReg2);\r
-\r
- updateValueMap(II, ResultReg, 2);\r
- return true;\r
- }\r
- case Intrinsic::x86_sse_cvttss2si:\r
- case Intrinsic::x86_sse_cvttss2si64:\r
- case Intrinsic::x86_sse2_cvttsd2si:\r
- case Intrinsic::x86_sse2_cvttsd2si64: {\r
- bool IsInputDouble;\r
- switch (II->getIntrinsicID()) {\r
- default: llvm_unreachable("Unexpected intrinsic.");\r
- case Intrinsic::x86_sse_cvttss2si:\r
- case Intrinsic::x86_sse_cvttss2si64:\r
- if (!Subtarget->hasSSE1())\r
- return false;\r
- IsInputDouble = false;\r
- break;\r
- case Intrinsic::x86_sse2_cvttsd2si:\r
- case Intrinsic::x86_sse2_cvttsd2si64:\r
- if (!Subtarget->hasSSE2())\r
- return false;\r
- IsInputDouble = true;\r
- break;\r
- }\r
-\r
- Type *RetTy = II->getCalledFunction()->getReturnType();\r
- MVT VT;\r
- if (!isTypeLegal(RetTy, VT))\r
- return false;\r
-\r
- static const unsigned CvtOpc[2][2][2] = {\r
- { { X86::CVTTSS2SIrr, X86::VCVTTSS2SIrr },\r
- { X86::CVTTSS2SI64rr, X86::VCVTTSS2SI64rr } },\r
- { { X86::CVTTSD2SIrr, X86::VCVTTSD2SIrr },\r
- { X86::CVTTSD2SI64rr, X86::VCVTTSD2SI64rr } }\r
- };\r
- bool HasAVX = Subtarget->hasAVX();\r
- unsigned Opc;\r
- switch (VT.SimpleTy) {\r
- default: llvm_unreachable("Unexpected result type.");\r
- case MVT::i32: Opc = CvtOpc[IsInputDouble][0][HasAVX]; break;\r
- case MVT::i64: Opc = CvtOpc[IsInputDouble][1][HasAVX]; break;\r
- }\r
-\r
- // Check if we can fold insertelement instructions into the convert.\r
- const Value *Op = II->getArgOperand(0);\r
- while (auto *IE = dyn_cast<InsertElementInst>(Op)) {\r
- const Value *Index = IE->getOperand(2);\r
- if (!isa<ConstantInt>(Index))\r
- break;\r
- unsigned Idx = cast<ConstantInt>(Index)->getZExtValue();\r
-\r
- if (Idx == 0) {\r
- Op = IE->getOperand(1);\r
- break;\r
- }\r
- Op = IE->getOperand(0);\r
- }\r
-\r
- unsigned Reg = getRegForValue(Op);\r
- if (Reg == 0)\r
- return false;\r
-\r
- unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)\r
- .addReg(Reg);\r
-\r
- updateValueMap(II, ResultReg);\r
- return true;\r
- }\r
- }\r
-}\r
-\r
-bool X86FastISel::fastLowerArguments() {\r
- if (!FuncInfo.CanLowerReturn)\r
- return false;\r
-\r
- const Function *F = FuncInfo.Fn;\r
- if (F->isVarArg())\r
- return false;\r
-\r
- CallingConv::ID CC = F->getCallingConv();\r
- if (CC != CallingConv::C)\r
- return false;\r
-\r
- if (Subtarget->isCallingConvWin64(CC))\r
- return false;\r
-\r
- if (!Subtarget->is64Bit())\r
- return false;\r
-\r
- // Only handle simple cases. i.e. Up to 6 i32/i64 scalar arguments.\r
- unsigned GPRCnt = 0;\r
- unsigned FPRCnt = 0;\r
- unsigned Idx = 0;\r
- for (auto const &Arg : F->args()) {\r
- // The first argument is at index 1.\r
- ++Idx;\r
- if (F->getAttributes().hasAttribute(Idx, Attribute::ByVal) ||\r
- F->getAttributes().hasAttribute(Idx, Attribute::InReg) ||\r
- F->getAttributes().hasAttribute(Idx, Attribute::StructRet) ||\r
- F->getAttributes().hasAttribute(Idx, Attribute::Nest))\r
- return false;\r
-\r
- Type *ArgTy = Arg.getType();\r
- if (ArgTy->isStructTy() || ArgTy->isArrayTy() || ArgTy->isVectorTy())\r
- return false;\r
-\r
- EVT ArgVT = TLI.getValueType(ArgTy);\r
- if (!ArgVT.isSimple()) return false;\r
- switch (ArgVT.getSimpleVT().SimpleTy) {\r
- default: return false;\r
- case MVT::i32:\r
- case MVT::i64:\r
- ++GPRCnt;\r
- break;\r
- case MVT::f32:\r
- case MVT::f64:\r
- if (!Subtarget->hasSSE1())\r
- return false;\r
- ++FPRCnt;\r
- break;\r
- }\r
-\r
- if (GPRCnt > 6)\r
- return false;\r
-\r
- if (FPRCnt > 8)\r
- return false;\r
- }\r
-\r
- static const MCPhysReg GPR32ArgRegs[] = {\r
- X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D\r
- };\r
- static const MCPhysReg GPR64ArgRegs[] = {\r
- X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8 , X86::R9\r
- };\r
- static const MCPhysReg XMMArgRegs[] = {\r
- X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,\r
- X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7\r
- };\r
-\r
- unsigned GPRIdx = 0;\r
- unsigned FPRIdx = 0;\r
- for (auto const &Arg : F->args()) {\r
- MVT VT = TLI.getSimpleValueType(Arg.getType());\r
- const TargetRegisterClass *RC = TLI.getRegClassFor(VT);\r
- unsigned SrcReg;\r
- switch (VT.SimpleTy) {\r
- default: llvm_unreachable("Unexpected value type.");\r
- case MVT::i32: SrcReg = GPR32ArgRegs[GPRIdx++]; break;\r
- case MVT::i64: SrcReg = GPR64ArgRegs[GPRIdx++]; break;\r
- case MVT::f32: // fall-through\r
- case MVT::f64: SrcReg = XMMArgRegs[FPRIdx++]; break;\r
- }\r
- unsigned DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);\r
- // FIXME: Unfortunately it's necessary to emit a copy from the livein copy.\r
- // Without this, EmitLiveInCopies may eliminate the livein if its only\r
- // use is a bitcast (which isn't turned into an instruction).\r
- unsigned ResultReg = createResultReg(RC);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::COPY), ResultReg)\r
- .addReg(DstReg, getKillRegState(true));\r
- updateValueMap(&Arg, ResultReg);\r
- }\r
- return true;\r
-}\r
-\r
-static unsigned computeBytesPoppedByCallee(const X86Subtarget *Subtarget,\r
- CallingConv::ID CC,\r
- ImmutableCallSite *CS) {\r
- if (Subtarget->is64Bit())\r
- return 0;\r
- if (Subtarget->getTargetTriple().isOSMSVCRT())\r
- return 0;\r
- if (CC == CallingConv::Fast || CC == CallingConv::GHC ||\r
- CC == CallingConv::HiPE)\r
- return 0;\r
- if (CS && !CS->paramHasAttr(1, Attribute::StructRet))\r
- return 0;\r
- if (CS && CS->paramHasAttr(1, Attribute::InReg))\r
- return 0;\r
- return 4;\r
-}\r
-\r
-bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {\r
- auto &OutVals = CLI.OutVals;\r
- auto &OutFlags = CLI.OutFlags;\r
- auto &OutRegs = CLI.OutRegs;\r
- auto &Ins = CLI.Ins;\r
- auto &InRegs = CLI.InRegs;\r
- CallingConv::ID CC = CLI.CallConv;\r
- bool &IsTailCall = CLI.IsTailCall;\r
- bool IsVarArg = CLI.IsVarArg;\r
- const Value *Callee = CLI.Callee;\r
- const char *SymName = CLI.SymName;\r
-\r
- bool Is64Bit = Subtarget->is64Bit();\r
- bool IsWin64 = Subtarget->isCallingConvWin64(CC);\r
-\r
- // Handle only C, fastcc, and webkit_js calling conventions for now.\r
- switch (CC) {\r
- default: return false;\r
- case CallingConv::C:\r
- case CallingConv::Fast:\r
- case CallingConv::WebKit_JS:\r
- case CallingConv::X86_FastCall:\r
- case CallingConv::X86_64_Win64:\r
- case CallingConv::X86_64_SysV:\r
- break;\r
- }\r
-\r
- // Allow SelectionDAG isel to handle tail calls.\r
- if (IsTailCall)\r
- return false;\r
-\r
- // fastcc with -tailcallopt is intended to provide a guaranteed\r
- // tail call optimization. Fastisel doesn't know how to do that.\r
- if (CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt)\r
- return false;\r
-\r
- // Don't know how to handle Win64 varargs yet. Nothing special needed for\r
- // x86-32. Special handling for x86-64 is implemented.\r
- if (IsVarArg && IsWin64)\r
- return false;\r
-\r
- // Don't know about inalloca yet.\r
- if (CLI.CS && CLI.CS->hasInAllocaArgument())\r
- return false;\r
-\r
- // Fast-isel doesn't know about callee-pop yet.\r
- if (X86::isCalleePop(CC, Subtarget->is64Bit(), IsVarArg,\r
- TM.Options.GuaranteedTailCallOpt))\r
- return false;\r
-\r
- SmallVector<MVT, 16> OutVTs;\r
- SmallVector<unsigned, 16> ArgRegs;\r
-\r
- // If this is a constant i1/i8/i16 argument, promote to i32 to avoid an extra\r
- // instruction. This is safe because it is common to all FastISel supported\r
- // calling conventions on x86.\r
- for (int i = 0, e = OutVals.size(); i != e; ++i) {\r
- Value *&Val = OutVals[i];\r
- ISD::ArgFlagsTy Flags = OutFlags[i];\r
- if (auto *CI = dyn_cast<ConstantInt>(Val)) {\r
- if (CI->getBitWidth() < 32) {\r
- if (Flags.isSExt())\r
- Val = ConstantExpr::getSExt(CI, Type::getInt32Ty(CI->getContext()));\r
- else\r
- Val = ConstantExpr::getZExt(CI, Type::getInt32Ty(CI->getContext()));\r
- }\r
- }\r
-\r
- // Passing bools around ends up doing a trunc to i1 and passing it.\r
- // Codegen this as an argument + "and 1".\r
- MVT VT;\r
- auto *TI = dyn_cast<TruncInst>(Val);\r
- unsigned ResultReg;\r
- if (TI && TI->getType()->isIntegerTy(1) && CLI.CS &&\r
- (TI->getParent() == CLI.CS->getInstruction()->getParent()) &&\r
- TI->hasOneUse()) {\r
- Value *PrevVal = TI->getOperand(0);\r
- ResultReg = getRegForValue(PrevVal);\r
-\r
- if (!ResultReg)\r
- return false;\r
-\r
- if (!isTypeLegal(PrevVal->getType(), VT))\r
- return false;\r
-\r
- ResultReg =\r
- fastEmit_ri(VT, VT, ISD::AND, ResultReg, hasTrivialKill(PrevVal), 1);\r
- } else {\r
- if (!isTypeLegal(Val->getType(), VT))\r
- return false;\r
- ResultReg = getRegForValue(Val);\r
- }\r
-\r
- if (!ResultReg)\r
- return false;\r
-\r
- ArgRegs.push_back(ResultReg);\r
- OutVTs.push_back(VT);\r
- }\r
-\r
- // Analyze operands of the call, assigning locations to each operand.\r
- SmallVector<CCValAssign, 16> ArgLocs;\r
- CCState CCInfo(CC, IsVarArg, *FuncInfo.MF, ArgLocs, CLI.RetTy->getContext());\r
-\r
- // Allocate shadow area for Win64\r
- if (IsWin64)\r
- CCInfo.AllocateStack(32, 8);\r
-\r
- CCInfo.AnalyzeCallOperands(OutVTs, OutFlags, CC_X86);\r
-\r
- // Get a count of how many bytes are to be pushed on the stack.\r
- unsigned NumBytes = CCInfo.getNextStackOffset();\r
-\r
- // Issue CALLSEQ_START\r
- unsigned AdjStackDown = TII.getCallFrameSetupOpcode();\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown))\r
- .addImm(NumBytes).addImm(0);\r
-\r
- // Walk the register/memloc assignments, inserting copies/loads.\r
- const X86RegisterInfo *RegInfo = static_cast<const X86RegisterInfo *>(\r
- TM.getSubtargetImpl()->getRegisterInfo());\r
- for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {\r
- CCValAssign const &VA = ArgLocs[i];\r
- const Value *ArgVal = OutVals[VA.getValNo()];\r
- MVT ArgVT = OutVTs[VA.getValNo()];\r
-\r
- if (ArgVT == MVT::x86mmx)\r
- return false;\r
-\r
- unsigned ArgReg = ArgRegs[VA.getValNo()];\r
-\r
- // Promote the value if needed.\r
- switch (VA.getLocInfo()) {\r
- case CCValAssign::Full: break;\r
- case CCValAssign::SExt: {\r
- assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&\r
- "Unexpected extend");\r
- bool Emitted = X86FastEmitExtend(ISD::SIGN_EXTEND, VA.getLocVT(), ArgReg,\r
- ArgVT, ArgReg);\r
- assert(Emitted && "Failed to emit a sext!"); (void)Emitted;\r
- ArgVT = VA.getLocVT();\r
- break;\r
- }\r
- case CCValAssign::ZExt: {\r
- assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&\r
- "Unexpected extend");\r
- bool Emitted = X86FastEmitExtend(ISD::ZERO_EXTEND, VA.getLocVT(), ArgReg,\r
- ArgVT, ArgReg);\r
- assert(Emitted && "Failed to emit a zext!"); (void)Emitted;\r
- ArgVT = VA.getLocVT();\r
- break;\r
- }\r
- case CCValAssign::AExt: {\r
- assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&\r
- "Unexpected extend");\r
- bool Emitted = X86FastEmitExtend(ISD::ANY_EXTEND, VA.getLocVT(), ArgReg,\r
- ArgVT, ArgReg);\r
- if (!Emitted)\r
- Emitted = X86FastEmitExtend(ISD::ZERO_EXTEND, VA.getLocVT(), ArgReg,\r
- ArgVT, ArgReg);\r
- if (!Emitted)\r
- Emitted = X86FastEmitExtend(ISD::SIGN_EXTEND, VA.getLocVT(), ArgReg,\r
- ArgVT, ArgReg);\r
-\r
- assert(Emitted && "Failed to emit a aext!"); (void)Emitted;\r
- ArgVT = VA.getLocVT();\r
- break;\r
- }\r
- case CCValAssign::BCvt: {\r
- ArgReg = fastEmit_r(ArgVT, VA.getLocVT(), ISD::BITCAST, ArgReg,\r
- /*TODO: Kill=*/false);\r
- assert(ArgReg && "Failed to emit a bitcast!");\r
- ArgVT = VA.getLocVT();\r
- break;\r
- }\r
- case CCValAssign::VExt:\r
- // VExt has not been implemented, so this should be impossible to reach\r
- // for now. However, fallback to Selection DAG isel once implemented.\r
- return false;\r
- case CCValAssign::AExtUpper:\r
- case CCValAssign::SExtUpper:\r
- case CCValAssign::ZExtUpper:\r
- case CCValAssign::FPExt:\r
- llvm_unreachable("Unexpected loc info!");\r
- case CCValAssign::Indirect:\r
- // FIXME: Indirect doesn't need extending, but fast-isel doesn't fully\r
- // support this.\r
- return false;\r
- }\r
-\r
- if (VA.isRegLoc()) {\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::COPY), VA.getLocReg()).addReg(ArgReg);\r
- OutRegs.push_back(VA.getLocReg());\r
- } else {\r
- assert(VA.isMemLoc());\r
-\r
- // Don't emit stores for undef values.\r
- if (isa<UndefValue>(ArgVal))\r
- continue;\r
-\r
- unsigned LocMemOffset = VA.getLocMemOffset();\r
- X86AddressMode AM;\r
- AM.Base.Reg = RegInfo->getStackRegister();\r
- AM.Disp = LocMemOffset;\r
- ISD::ArgFlagsTy Flags = OutFlags[VA.getValNo()];\r
- unsigned Alignment = DL.getABITypeAlignment(ArgVal->getType());\r
- MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(\r
- MachinePointerInfo::getStack(LocMemOffset), MachineMemOperand::MOStore,\r
- ArgVT.getStoreSize(), Alignment);\r
- if (Flags.isByVal()) {\r
- X86AddressMode SrcAM;\r
- SrcAM.Base.Reg = ArgReg;\r
- if (!TryEmitSmallMemcpy(AM, SrcAM, Flags.getByValSize()))\r
- return false;\r
- } else if (isa<ConstantInt>(ArgVal) || isa<ConstantPointerNull>(ArgVal)) {\r
- // If this is a really simple value, emit this with the Value* version\r
- // of X86FastEmitStore. If it isn't simple, we don't want to do this,\r
- // as it can cause us to reevaluate the argument.\r
- if (!X86FastEmitStore(ArgVT, ArgVal, AM, MMO))\r
- return false;\r
- } else {\r
- bool ValIsKill = hasTrivialKill(ArgVal);\r
- if (!X86FastEmitStore(ArgVT, ArgReg, ValIsKill, AM, MMO))\r
- return false;\r
- }\r
- }\r
- }\r
-\r
- // ELF / PIC requires GOT in the EBX register before function calls via PLT\r
- // GOT pointer.\r
- if (Subtarget->isPICStyleGOT()) {\r
- unsigned Base = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::COPY), X86::EBX).addReg(Base);\r
- }\r
-\r
- if (Is64Bit && IsVarArg && !IsWin64) {\r
- // From AMD64 ABI document:\r
- // For calls that may call functions that use varargs or stdargs\r
- // (prototype-less calls or calls to functions containing ellipsis (...) in\r
- // the declaration) %al is used as hidden argument to specify the number\r
- // of SSE registers used. The contents of %al do not need to match exactly\r
- // the number of registers, but must be an ubound on the number of SSE\r
- // registers used and is in the range 0 - 8 inclusive.\r
-\r
- // Count the number of XMM registers allocated.\r
- static const MCPhysReg XMMArgRegs[] = {\r
- X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,\r
- X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7\r
- };\r
- unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8);\r
- assert((Subtarget->hasSSE1() || !NumXMMRegs)\r
- && "SSE registers cannot be used when SSE is disabled");\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV8ri),\r
- X86::AL).addImm(NumXMMRegs);\r
- }\r
-\r
- // Materialize callee address in a register. FIXME: GV address can be\r
- // handled with a CALLpcrel32 instead.\r
- X86AddressMode CalleeAM;\r
- if (!X86SelectCallAddress(Callee, CalleeAM))\r
- return false;\r
-\r
- unsigned CalleeOp = 0;\r
- const GlobalValue *GV = nullptr;\r
- if (CalleeAM.GV != nullptr) {\r
- GV = CalleeAM.GV;\r
- } else if (CalleeAM.Base.Reg != 0) {\r
- CalleeOp = CalleeAM.Base.Reg;\r
- } else\r
- return false;\r
-\r
- // Issue the call.\r
- MachineInstrBuilder MIB;\r
- if (CalleeOp) {\r
- // Register-indirect call.\r
- unsigned CallOpc = Is64Bit ? X86::CALL64r : X86::CALL32r;\r
- MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CallOpc))\r
- .addReg(CalleeOp);\r
- } else {\r
- // Direct call.\r
- assert(GV && "Not a direct call");\r
- unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;\r
-\r
- // See if we need any target-specific flags on the GV operand.\r
- unsigned char OpFlags = 0;\r
-\r
- // On ELF targets, in both X86-64 and X86-32 mode, direct calls to\r
- // external symbols most go through the PLT in PIC mode. If the symbol\r
- // has hidden or protected visibility, or if it is static or local, then\r
- // we don't need to use the PLT - we can directly call it.\r
- if (Subtarget->isTargetELF() &&\r
- TM.getRelocationModel() == Reloc::PIC_ &&\r
- GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) {\r
- OpFlags = X86II::MO_PLT;\r
- } else if (Subtarget->isPICStyleStubAny() &&\r
- (GV->isDeclaration() || GV->isWeakForLinker()) &&\r
- (!Subtarget->getTargetTriple().isMacOSX() ||\r
- Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) {\r
- // PC-relative references to external symbols should go through $stub,\r
- // unless we're building with the leopard linker or later, which\r
- // automatically synthesizes these stubs.\r
- OpFlags = X86II::MO_DARWIN_STUB;\r
- }\r
-\r
- MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CallOpc));\r
- if (SymName)\r
- MIB.addExternalSymbol(SymName, OpFlags);\r
- else\r
- MIB.addGlobalAddress(GV, 0, OpFlags);\r
- }\r
-\r
- // Add a register mask operand representing the call-preserved registers.\r
- // Proper defs for return values will be added by setPhysRegsDeadExcept().\r
- MIB.addRegMask(TRI.getCallPreservedMask(CC));\r
-\r
- // Add an implicit use GOT pointer in EBX.\r
- if (Subtarget->isPICStyleGOT())\r
- MIB.addReg(X86::EBX, RegState::Implicit);\r
-\r
- if (Is64Bit && IsVarArg && !IsWin64)\r
- MIB.addReg(X86::AL, RegState::Implicit);\r
-\r
- // Add implicit physical register uses to the call.\r
- for (auto Reg : OutRegs)\r
- MIB.addReg(Reg, RegState::Implicit);\r
-\r
- // Issue CALLSEQ_END\r
- unsigned NumBytesForCalleeToPop =\r
- computeBytesPoppedByCallee(Subtarget, CC, CLI.CS);\r
- unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))\r
- .addImm(NumBytes).addImm(NumBytesForCalleeToPop);\r
-\r
- // Now handle call return values.\r
- SmallVector<CCValAssign, 16> RVLocs;\r
- CCState CCRetInfo(CC, IsVarArg, *FuncInfo.MF, RVLocs,\r
- CLI.RetTy->getContext());\r
- CCRetInfo.AnalyzeCallResult(Ins, RetCC_X86);\r
-\r
- // Copy all of the result registers out of their specified physreg.\r
- unsigned ResultReg = FuncInfo.CreateRegs(CLI.RetTy);\r
- for (unsigned i = 0; i != RVLocs.size(); ++i) {\r
- CCValAssign &VA = RVLocs[i];\r
- EVT CopyVT = VA.getValVT();\r
- unsigned CopyReg = ResultReg + i;\r
-\r
- // If this is x86-64, and we disabled SSE, we can't return FP values\r
- if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&\r
- ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) {\r
- report_fatal_error("SSE register return with SSE disabled");\r
- }\r
-\r
- // If we prefer to use the value in xmm registers, copy it out as f80 and\r
- // use a truncate to move it from fp stack reg to xmm reg.\r
- if ((VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1) &&\r
- isScalarFPTypeInSSEReg(VA.getValVT())) {\r
- CopyVT = MVT::f80;\r
- CopyReg = createResultReg(&X86::RFP80RegClass);\r
- }\r
-\r
- // Copy out the result.\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::COPY), CopyReg).addReg(VA.getLocReg());\r
- InRegs.push_back(VA.getLocReg());\r
-\r
- // Round the f80 to the right size, which also moves it to the appropriate\r
- // xmm register. This is accomplished by storing the f80 value in memory\r
- // and then loading it back.\r
- if (CopyVT != VA.getValVT()) {\r
- EVT ResVT = VA.getValVT();\r
- unsigned Opc = ResVT == MVT::f32 ? X86::ST_Fp80m32 : X86::ST_Fp80m64;\r
- unsigned MemSize = ResVT.getSizeInBits()/8;\r
- int FI = MFI.CreateStackObject(MemSize, MemSize, false);\r
- addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Opc)), FI)\r
- .addReg(CopyReg);\r
- Opc = ResVT == MVT::f32 ? X86::MOVSSrm : X86::MOVSDrm;\r
- addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Opc), ResultReg + i), FI);\r
- }\r
- }\r
-\r
- CLI.ResultReg = ResultReg;\r
- CLI.NumResultRegs = RVLocs.size();\r
- CLI.Call = MIB;\r
-\r
- return true;\r
-}\r
-\r
-bool\r
-X86FastISel::fastSelectInstruction(const Instruction *I) {\r
- switch (I->getOpcode()) {\r
- default: break;\r
- case Instruction::Load:\r
- return X86SelectLoad(I);\r
- case Instruction::Store:\r
- return X86SelectStore(I);\r
- case Instruction::Ret:\r
- return X86SelectRet(I);\r
- case Instruction::ICmp:\r
- case Instruction::FCmp:\r
- return X86SelectCmp(I);\r
- case Instruction::ZExt:\r
- return X86SelectZExt(I);\r
- case Instruction::Br:\r
- return X86SelectBranch(I);\r
- case Instruction::LShr:\r
- case Instruction::AShr:\r
- case Instruction::Shl:\r
- return X86SelectShift(I);\r
- case Instruction::SDiv:\r
- case Instruction::UDiv:\r
- case Instruction::SRem:\r
- case Instruction::URem:\r
- return X86SelectDivRem(I);\r
- case Instruction::Select:\r
- return X86SelectSelect(I);\r
- case Instruction::Trunc:\r
- return X86SelectTrunc(I);\r
- case Instruction::FPExt:\r
- return X86SelectFPExt(I);\r
- case Instruction::FPTrunc:\r
- return X86SelectFPTrunc(I);\r
- case Instruction::IntToPtr: // Deliberate fall-through.\r
- case Instruction::PtrToInt: {\r
- EVT SrcVT = TLI.getValueType(I->getOperand(0)->getType());\r
- EVT DstVT = TLI.getValueType(I->getType());\r
- if (DstVT.bitsGT(SrcVT))\r
- return X86SelectZExt(I);\r
- if (DstVT.bitsLT(SrcVT))\r
- return X86SelectTrunc(I);\r
- unsigned Reg = getRegForValue(I->getOperand(0));\r
- if (Reg == 0) return false;\r
- updateValueMap(I, Reg);\r
- return true;\r
- }\r
- }\r
-\r
- return false;\r
-}\r
-\r
-unsigned X86FastISel::X86MaterializeInt(const ConstantInt *CI, MVT VT) {\r
- if (VT > MVT::i64)\r
- return 0;\r
-\r
- uint64_t Imm = CI->getZExtValue();\r
- if (Imm == 0) {\r
- unsigned SrcReg = fastEmitInst_(X86::MOV32r0, &X86::GR32RegClass);\r
- switch (VT.SimpleTy) {\r
- default: llvm_unreachable("Unexpected value type");\r
- case MVT::i1:\r
- case MVT::i8:\r
- return fastEmitInst_extractsubreg(MVT::i8, SrcReg, /*Kill=*/true,\r
- X86::sub_8bit);\r
- case MVT::i16:\r
- return fastEmitInst_extractsubreg(MVT::i16, SrcReg, /*Kill=*/true,\r
- X86::sub_16bit);\r
- case MVT::i32:\r
- return SrcReg;\r
- case MVT::i64: {\r
- unsigned ResultReg = createResultReg(&X86::GR64RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::SUBREG_TO_REG), ResultReg)\r
- .addImm(0).addReg(SrcReg).addImm(X86::sub_32bit);\r
- return ResultReg;\r
- }\r
- }\r
- }\r
-\r
- unsigned Opc = 0;\r
- switch (VT.SimpleTy) {\r
- default: llvm_unreachable("Unexpected value type");\r
- case MVT::i1: VT = MVT::i8; // fall-through\r
- case MVT::i8: Opc = X86::MOV8ri; break;\r
- case MVT::i16: Opc = X86::MOV16ri; break;\r
- case MVT::i32: Opc = X86::MOV32ri; break;\r
- case MVT::i64: {\r
- if (isUInt<32>(Imm))\r
- Opc = X86::MOV32ri;\r
- else if (isInt<32>(Imm))\r
- Opc = X86::MOV64ri32;\r
- else\r
- Opc = X86::MOV64ri;\r
- break;\r
- }\r
- }\r
- if (VT == MVT::i64 && Opc == X86::MOV32ri) {\r
- unsigned SrcReg = fastEmitInst_i(Opc, &X86::GR32RegClass, Imm);\r
- unsigned ResultReg = createResultReg(&X86::GR64RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(TargetOpcode::SUBREG_TO_REG), ResultReg)\r
- .addImm(0).addReg(SrcReg).addImm(X86::sub_32bit);\r
- return ResultReg;\r
- }\r
- return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);\r
-}\r
-\r
-unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {\r
- if (CFP->isNullValue())\r
- return fastMaterializeFloatZero(CFP);\r
-\r
- // Can't handle alternate code models yet.\r
- CodeModel::Model CM = TM.getCodeModel();\r
- if (CM != CodeModel::Small && CM != CodeModel::Large)\r
- return 0;\r
-\r
- // Get opcode and regclass of the output for the given load instruction.\r
- unsigned Opc = 0;\r
- const TargetRegisterClass *RC = nullptr;\r
- switch (VT.SimpleTy) {\r
- default: return 0;\r
- case MVT::f32:\r
- if (X86ScalarSSEf32) {\r
- Opc = Subtarget->hasAVX() ? X86::VMOVSSrm : X86::MOVSSrm;\r
- RC = &X86::FR32RegClass;\r
- } else {\r
- Opc = X86::LD_Fp32m;\r
- RC = &X86::RFP32RegClass;\r
- }\r
- break;\r
- case MVT::f64:\r
- if (X86ScalarSSEf64) {\r
- Opc = Subtarget->hasAVX() ? X86::VMOVSDrm : X86::MOVSDrm;\r
- RC = &X86::FR64RegClass;\r
- } else {\r
- Opc = X86::LD_Fp64m;\r
- RC = &X86::RFP64RegClass;\r
- }\r
- break;\r
- case MVT::f80:\r
- // No f80 support yet.\r
- return 0;\r
- }\r
-\r
- // MachineConstantPool wants an explicit alignment.\r
- unsigned Align = DL.getPrefTypeAlignment(CFP->getType());\r
- if (Align == 0) {\r
- // Alignment of vector types. FIXME!\r
- Align = DL.getTypeAllocSize(CFP->getType());\r
- }\r
-\r
- // x86-32 PIC requires a PIC base register for constant pools.\r
- unsigned PICBase = 0;\r
- unsigned char OpFlag = 0;\r
- if (Subtarget->isPICStyleStubPIC()) { // Not dynamic-no-pic\r
- OpFlag = X86II::MO_PIC_BASE_OFFSET;\r
- PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);\r
- } else if (Subtarget->isPICStyleGOT()) {\r
- OpFlag = X86II::MO_GOTOFF;\r
- PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);\r
- } else if (Subtarget->isPICStyleRIPRel() &&\r
- TM.getCodeModel() == CodeModel::Small) {\r
- PICBase = X86::RIP;\r
- }\r
-\r
- // Create the load from the constant pool.\r
- unsigned CPI = MCP.getConstantPoolIndex(CFP, Align);\r
- unsigned ResultReg = createResultReg(RC);\r
-\r
- if (CM == CodeModel::Large) {\r
- unsigned AddrReg = createResultReg(&X86::GR64RegClass);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV64ri),\r
- AddrReg)\r
- .addConstantPoolIndex(CPI, 0, OpFlag);\r
- MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Opc), ResultReg);\r
- addDirectMem(MIB, AddrReg);\r
- MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(\r
- MachinePointerInfo::getConstantPool(), MachineMemOperand::MOLoad,\r
- TM.getDataLayout()->getPointerSize(), Align);\r
- MIB->addMemOperand(*FuncInfo.MF, MMO);\r
- return ResultReg;\r
- }\r
-\r
- addConstantPoolReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Opc), ResultReg),\r
- CPI, PICBase, OpFlag);\r
- return ResultReg;\r
-}\r
-\r
-unsigned X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {\r
- // Can't handle alternate code models yet.\r
- if (TM.getCodeModel() != CodeModel::Small)\r
- return 0;\r
-\r
- // Materialize addresses with LEA/MOV instructions.\r
- X86AddressMode AM;\r
- if (X86SelectAddress(GV, AM)) {\r
- // If the expression is just a basereg, then we're done, otherwise we need\r
- // to emit an LEA.\r
- if (AM.BaseType == X86AddressMode::RegBase &&\r
- AM.IndexReg == 0 && AM.Disp == 0 && AM.GV == nullptr)\r
- return AM.Base.Reg;\r
-\r
- unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));\r
- if (TM.getRelocationModel() == Reloc::Static &&\r
- TLI.getPointerTy() == MVT::i64) {\r
- // The displacement code could be more than 32 bits away so we need to use\r
- // an instruction with a 64 bit immediate\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV64ri),\r
- ResultReg)\r
- .addGlobalAddress(GV);\r
- } else {\r
- unsigned Opc = TLI.getPointerTy() == MVT::i32\r
- ? (Subtarget->isTarget64BitILP32()\r
- ? X86::LEA64_32r : X86::LEA32r)\r
- : X86::LEA64r;\r
- addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Opc), ResultReg), AM);\r
- }\r
- return ResultReg;\r
- }\r
- return 0;\r
-}\r
-\r
-unsigned X86FastISel::fastMaterializeConstant(const Constant *C) {\r
- EVT CEVT = TLI.getValueType(C->getType(), true);\r
-\r
- // Only handle simple types.\r
- if (!CEVT.isSimple())\r
- return 0;\r
- MVT VT = CEVT.getSimpleVT();\r
-\r
- if (const auto *CI = dyn_cast<ConstantInt>(C))\r
- return X86MaterializeInt(CI, VT);\r
- else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))\r
- return X86MaterializeFP(CFP, VT);\r
- else if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))\r
- return X86MaterializeGV(GV, VT);\r
-\r
- return 0;\r
-}\r
-\r
-unsigned X86FastISel::fastMaterializeAlloca(const AllocaInst *C) {\r
- // Fail on dynamic allocas. At this point, getRegForValue has already\r
- // checked its CSE maps, so if we're here trying to handle a dynamic\r
- // alloca, we're not going to succeed. X86SelectAddress has a\r
- // check for dynamic allocas, because it's called directly from\r
- // various places, but targetMaterializeAlloca also needs a check\r
- // in order to avoid recursion between getRegForValue,\r
- // X86SelectAddrss, and targetMaterializeAlloca.\r
- if (!FuncInfo.StaticAllocaMap.count(C))\r
- return 0;\r
- assert(C->isStaticAlloca() && "dynamic alloca in the static alloca map?");\r
-\r
- X86AddressMode AM;\r
- if (!X86SelectAddress(C, AM))\r
- return 0;\r
- unsigned Opc = TLI.getPointerTy() == MVT::i32\r
- ? (Subtarget->isTarget64BitILP32()\r
- ? X86::LEA64_32r : X86::LEA32r)\r
- : X86::LEA64r;\r
- const TargetRegisterClass* RC = TLI.getRegClassFor(TLI.getPointerTy());\r
- unsigned ResultReg = createResultReg(RC);\r
- addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,\r
- TII.get(Opc), ResultReg), AM);\r
- return ResultReg;\r
-}\r
-\r
-unsigned X86FastISel::fastMaterializeFloatZero(const ConstantFP *CF) {\r
- MVT VT;\r
- if (!isTypeLegal(CF->getType(), VT))\r
- return 0;\r
-\r
- // Get opcode and regclass for the given zero.\r
- unsigned Opc = 0;\r
- const TargetRegisterClass *RC = nullptr;\r
- switch (VT.SimpleTy) {\r
- default: return 0;\r
- case MVT::f32:\r
- if (X86ScalarSSEf32) {\r
- Opc = X86::FsFLD0SS;\r
- RC = &X86::FR32RegClass;\r
- } else {\r
- Opc = X86::LD_Fp032;\r
- RC = &X86::RFP32RegClass;\r
- }\r
- break;\r
- case MVT::f64:\r
- if (X86ScalarSSEf64) {\r
- Opc = X86::FsFLD0SD;\r
- RC = &X86::FR64RegClass;\r
- } else {\r
- Opc = X86::LD_Fp064;\r
- RC = &X86::RFP64RegClass;\r
- }\r
- break;\r
- case MVT::f80:\r
- // No f80 support yet.\r
- return 0;\r
- }\r
-\r
- unsigned ResultReg = createResultReg(RC);\r
- BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg);\r
- return ResultReg;\r
-}\r
-\r
-\r
-bool X86FastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,\r
- const LoadInst *LI) {\r
- const Value *Ptr = LI->getPointerOperand();\r
- X86AddressMode AM;\r
- if (!X86SelectAddress(Ptr, AM))\r
- return false;\r
-\r
- const X86InstrInfo &XII = (const X86InstrInfo &)TII;\r
-\r
- unsigned Size = DL.getTypeAllocSize(LI->getType());\r
- unsigned Alignment = LI->getAlignment();\r
-\r
- if (Alignment == 0) // Ensure that codegen never sees alignment 0\r
- Alignment = DL.getABITypeAlignment(LI->getType());\r
-\r
- SmallVector<MachineOperand, 8> AddrOps;\r
- AM.getFullAddress(AddrOps);\r
-\r
- MachineInstr *Result =\r
- XII.foldMemoryOperandImpl(*FuncInfo.MF, MI, OpNo, AddrOps,\r
- Size, Alignment, /*AllowCommute=*/true);\r
- if (!Result)\r
- return false;\r
-\r
- Result->addMemOperand(*FuncInfo.MF, createMachineMemOperandFor(LI));\r
- FuncInfo.MBB->insert(FuncInfo.InsertPt, Result);\r
- MI->eraseFromParent();\r
- return true;\r
-}\r
-\r
-\r
-namespace llvm {\r
- FastISel *X86::createFastISel(FunctionLoweringInfo &funcInfo,\r
- const TargetLibraryInfo *libInfo) {\r
- return new X86FastISel(funcInfo, libInfo);\r
- }\r
-}\r
+//===-- X86FastISel.cpp - X86 FastISel implementation ---------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the X86-specific support for the FastISel class. Much
+// of the target-specific code is generated by tablegen in the file
+// X86GenFastISel.inc, which is #included here.
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86.h"
+#include "X86CallingConv.h"
+#include "X86InstrBuilder.h"
+#include "X86InstrInfo.h"
+#include "X86MachineFunctionInfo.h"
+#include "X86RegisterInfo.h"
+#include "X86Subtarget.h"
+#include "X86TargetMachine.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/CodeGen/Analysis.h"
+#include "llvm/CodeGen/FastISel.h"
+#include "llvm/CodeGen/FunctionLoweringInfo.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/CallSite.h"
+#include "llvm/IR/CallingConv.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/GetElementPtrTypeIterator.h"
+#include "llvm/IR/GlobalAlias.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Target/TargetOptions.h"
+using namespace llvm;
+
+namespace {
+
+class X86FastISel final : public FastISel {
+ /// Subtarget - Keep a pointer to the X86Subtarget around so that we can
+ /// make the right decision when generating code for different targets.
+ const X86Subtarget *Subtarget;
+
+ /// X86ScalarSSEf32, X86ScalarSSEf64 - Select between SSE or x87
+ /// floating point ops.
+ /// When SSE is available, use it for f32 operations.
+ /// When SSE2 is available, use it for f64 operations.
+ bool X86ScalarSSEf64;
+ bool X86ScalarSSEf32;
+
+public:
+ explicit X86FastISel(FunctionLoweringInfo &funcInfo,
+ const TargetLibraryInfo *libInfo)
+ : FastISel(funcInfo, libInfo) {
+ Subtarget = &funcInfo.MF->getSubtarget<X86Subtarget>();
+ X86ScalarSSEf64 = Subtarget->hasSSE2();
+ X86ScalarSSEf32 = Subtarget->hasSSE1();
+ }
+
+ bool fastSelectInstruction(const Instruction *I) override;
+
+ /// \brief The specified machine instr operand is a vreg, and that
+ /// vreg is being provided by the specified load instruction. If possible,
+ /// try to fold the load as an operand to the instruction, returning true if
+ /// possible.
+ bool tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
+ const LoadInst *LI) override;
+
+ bool fastLowerArguments() override;
+ bool fastLowerCall(CallLoweringInfo &CLI) override;
+ bool fastLowerIntrinsicCall(const IntrinsicInst *II) override;
+
+#include "X86GenFastISel.inc"
+
+private:
+ bool X86FastEmitCompare(const Value *LHS, const Value *RHS, EVT VT, DebugLoc DL);
+
+ bool X86FastEmitLoad(EVT VT, X86AddressMode &AM, MachineMemOperand *MMO,
+ unsigned &ResultReg, unsigned Alignment = 1);
+
+ bool X86FastEmitStore(EVT VT, const Value *Val, X86AddressMode &AM,
+ MachineMemOperand *MMO = nullptr, bool Aligned = false);
+ bool X86FastEmitStore(EVT VT, unsigned ValReg, bool ValIsKill,
+ X86AddressMode &AM,
+ MachineMemOperand *MMO = nullptr, bool Aligned = false);
+
+ bool X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, unsigned Src, EVT SrcVT,
+ unsigned &ResultReg);
+
+ bool X86SelectAddress(const Value *V, X86AddressMode &AM);
+ bool X86SelectCallAddress(const Value *V, X86AddressMode &AM);
+
+ bool X86SelectLoad(const Instruction *I);
+
+ bool X86SelectStore(const Instruction *I);
+
+ bool X86SelectRet(const Instruction *I);
+
+ bool X86SelectCmp(const Instruction *I);
+
+ bool X86SelectZExt(const Instruction *I);
+
+ bool X86SelectBranch(const Instruction *I);
+
+ bool X86SelectShift(const Instruction *I);
+
+ bool X86SelectDivRem(const Instruction *I);
+
+ bool X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I);
+
+ bool X86FastEmitSSESelect(MVT RetVT, const Instruction *I);
+
+ bool X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I);
+
+ bool X86SelectSelect(const Instruction *I);
+
+ bool X86SelectTrunc(const Instruction *I);
+
+ bool X86SelectFPExtOrFPTrunc(const Instruction *I, unsigned Opc,
+ const TargetRegisterClass *RC);
+
+ bool X86SelectFPExt(const Instruction *I);
+ bool X86SelectFPTrunc(const Instruction *I);
+ bool X86SelectSIToFP(const Instruction *I);
+
+ const X86InstrInfo *getInstrInfo() const {
+ return Subtarget->getInstrInfo();
+ }
+ const X86TargetMachine *getTargetMachine() const {
+ return static_cast<const X86TargetMachine *>(&TM);
+ }
+
+ bool handleConstantAddresses(const Value *V, X86AddressMode &AM);
+
+ unsigned X86MaterializeInt(const ConstantInt *CI, MVT VT);
+ unsigned X86MaterializeFP(const ConstantFP *CFP, MVT VT);
+ unsigned X86MaterializeGV(const GlobalValue *GV, MVT VT);
+ unsigned fastMaterializeConstant(const Constant *C) override;
+
+ unsigned fastMaterializeAlloca(const AllocaInst *C) override;
+
+ unsigned fastMaterializeFloatZero(const ConstantFP *CF) override;
+
+ /// isScalarFPTypeInSSEReg - Return true if the specified scalar FP type is
+ /// computed in an SSE register, not on the X87 floating point stack.
+ bool isScalarFPTypeInSSEReg(EVT VT) const {
+ return (VT == MVT::f64 && X86ScalarSSEf64) || // f64 is when SSE2
+ (VT == MVT::f32 && X86ScalarSSEf32); // f32 is when SSE1
+ }
+
+ bool isTypeLegal(Type *Ty, MVT &VT, bool AllowI1 = false);
+
+ bool IsMemcpySmall(uint64_t Len);
+
+ bool TryEmitSmallMemcpy(X86AddressMode DestAM,
+ X86AddressMode SrcAM, uint64_t Len);
+
+ bool foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
+ const Value *Cond);
+
+ const MachineInstrBuilder &addFullAddress(const MachineInstrBuilder &MIB,
+ X86AddressMode &AM);
+};
+
+} // end anonymous namespace.
+
+static std::pair<X86::CondCode, bool>
+getX86ConditionCode(CmpInst::Predicate Predicate) {
+ X86::CondCode CC = X86::COND_INVALID;
+ bool NeedSwap = false;
+ switch (Predicate) {
+ default: break;
+ // Floating-point Predicates
+ case CmpInst::FCMP_UEQ: CC = X86::COND_E; break;
+ case CmpInst::FCMP_OLT: NeedSwap = true; // fall-through
+ case CmpInst::FCMP_OGT: CC = X86::COND_A; break;
+ case CmpInst::FCMP_OLE: NeedSwap = true; // fall-through
+ case CmpInst::FCMP_OGE: CC = X86::COND_AE; break;
+ case CmpInst::FCMP_UGT: NeedSwap = true; // fall-through
+ case CmpInst::FCMP_ULT: CC = X86::COND_B; break;
+ case CmpInst::FCMP_UGE: NeedSwap = true; // fall-through
+ case CmpInst::FCMP_ULE: CC = X86::COND_BE; break;
+ case CmpInst::FCMP_ONE: CC = X86::COND_NE; break;
+ case CmpInst::FCMP_UNO: CC = X86::COND_P; break;
+ case CmpInst::FCMP_ORD: CC = X86::COND_NP; break;
+ case CmpInst::FCMP_OEQ: // fall-through
+ case CmpInst::FCMP_UNE: CC = X86::COND_INVALID; break;
+
+ // Integer Predicates
+ case CmpInst::ICMP_EQ: CC = X86::COND_E; break;
+ case CmpInst::ICMP_NE: CC = X86::COND_NE; break;
+ case CmpInst::ICMP_UGT: CC = X86::COND_A; break;
+ case CmpInst::ICMP_UGE: CC = X86::COND_AE; break;
+ case CmpInst::ICMP_ULT: CC = X86::COND_B; break;
+ case CmpInst::ICMP_ULE: CC = X86::COND_BE; break;
+ case CmpInst::ICMP_SGT: CC = X86::COND_G; break;
+ case CmpInst::ICMP_SGE: CC = X86::COND_GE; break;
+ case CmpInst::ICMP_SLT: CC = X86::COND_L; break;
+ case CmpInst::ICMP_SLE: CC = X86::COND_LE; break;
+ }
+
+ return std::make_pair(CC, NeedSwap);
+}
+
+static std::pair<unsigned, bool>
+getX86SSEConditionCode(CmpInst::Predicate Predicate) {
+ unsigned CC;
+ bool NeedSwap = false;
+
+ // SSE Condition code mapping:
+ // 0 - EQ
+ // 1 - LT
+ // 2 - LE
+ // 3 - UNORD
+ // 4 - NEQ
+ // 5 - NLT
+ // 6 - NLE
+ // 7 - ORD
+ switch (Predicate) {
+ default: llvm_unreachable("Unexpected predicate");
+ case CmpInst::FCMP_OEQ: CC = 0; break;
+ case CmpInst::FCMP_OGT: NeedSwap = true; // fall-through
+ case CmpInst::FCMP_OLT: CC = 1; break;
+ case CmpInst::FCMP_OGE: NeedSwap = true; // fall-through
+ case CmpInst::FCMP_OLE: CC = 2; break;
+ case CmpInst::FCMP_UNO: CC = 3; break;
+ case CmpInst::FCMP_UNE: CC = 4; break;
+ case CmpInst::FCMP_ULE: NeedSwap = true; // fall-through
+ case CmpInst::FCMP_UGE: CC = 5; break;
+ case CmpInst::FCMP_ULT: NeedSwap = true; // fall-through
+ case CmpInst::FCMP_UGT: CC = 6; break;
+ case CmpInst::FCMP_ORD: CC = 7; break;
+ case CmpInst::FCMP_UEQ:
+ case CmpInst::FCMP_ONE: CC = 8; break;
+ }
+
+ return std::make_pair(CC, NeedSwap);
+}
+
+/// \brief Adds a complex addressing mode to the given machine instr builder.
+/// Note, this will constrain the index register. If its not possible to
+/// constrain the given index register, then a new one will be created. The
+/// IndexReg field of the addressing mode will be updated to match in this case.
+const MachineInstrBuilder &
+X86FastISel::addFullAddress(const MachineInstrBuilder &MIB,
+ X86AddressMode &AM) {
+ // First constrain the index register. It needs to be a GR64_NOSP.
+ AM.IndexReg = constrainOperandRegClass(MIB->getDesc(), AM.IndexReg,
+ MIB->getNumOperands() +
+ X86::AddrIndexReg);
+ return ::addFullAddress(MIB, AM);
+}
+
+/// \brief Check if it is possible to fold the condition from the XALU intrinsic
+/// into the user. The condition code will only be updated on success.
+bool X86FastISel::foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
+ const Value *Cond) {
+ if (!isa<ExtractValueInst>(Cond))
+ return false;
+
+ const auto *EV = cast<ExtractValueInst>(Cond);
+ if (!isa<IntrinsicInst>(EV->getAggregateOperand()))
+ return false;
+
+ const auto *II = cast<IntrinsicInst>(EV->getAggregateOperand());
+ MVT RetVT;
+ const Function *Callee = II->getCalledFunction();
+ Type *RetTy =
+ cast<StructType>(Callee->getReturnType())->getTypeAtIndex(0U);
+ if (!isTypeLegal(RetTy, RetVT))
+ return false;
+
+ if (RetVT != MVT::i32 && RetVT != MVT::i64)
+ return false;
+
+ X86::CondCode TmpCC;
+ switch (II->getIntrinsicID()) {
+ default: return false;
+ case Intrinsic::sadd_with_overflow:
+ case Intrinsic::ssub_with_overflow:
+ case Intrinsic::smul_with_overflow:
+ case Intrinsic::umul_with_overflow: TmpCC = X86::COND_O; break;
+ case Intrinsic::uadd_with_overflow:
+ case Intrinsic::usub_with_overflow: TmpCC = X86::COND_B; break;
+ }
+
+ // Check if both instructions are in the same basic block.
+ if (II->getParent() != I->getParent())
+ return false;
+
+ // Make sure nothing is in the way
+ BasicBlock::const_iterator Start = I;
+ BasicBlock::const_iterator End = II;
+ for (auto Itr = std::prev(Start); Itr != End; --Itr) {
+ // We only expect extractvalue instructions between the intrinsic and the
+ // instruction to be selected.
+ if (!isa<ExtractValueInst>(Itr))
+ return false;
+
+ // Check that the extractvalue operand comes from the intrinsic.
+ const auto *EVI = cast<ExtractValueInst>(Itr);
+ if (EVI->getAggregateOperand() != II)
+ return false;
+ }
+
+ CC = TmpCC;
+ return true;
+}
+
+bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {
+ EVT evt = TLI.getValueType(DL, Ty, /*HandleUnknown=*/true);
+ if (evt == MVT::Other || !evt.isSimple())
+ // Unhandled type. Halt "fast" selection and bail.
+ return false;
+
+ VT = evt.getSimpleVT();
+ // For now, require SSE/SSE2 for performing floating-point operations,
+ // since x87 requires additional work.
+ if (VT == MVT::f64 && !X86ScalarSSEf64)
+ return false;
+ if (VT == MVT::f32 && !X86ScalarSSEf32)
+ return false;
+ // Similarly, no f80 support yet.
+ if (VT == MVT::f80)
+ return false;
+ // We only handle legal types. For example, on x86-32 the instruction
+ // selector contains all of the 64-bit instructions from x86-64,
+ // under the assumption that i64 won't be used if the target doesn't
+ // support it.
+ return (AllowI1 && VT == MVT::i1) || TLI.isTypeLegal(VT);
+}
+
+#include "X86GenCallingConv.inc"
+
+/// X86FastEmitLoad - Emit a machine instruction to load a value of type VT.
+/// The address is either pre-computed, i.e. Ptr, or a GlobalAddress, i.e. GV.
+/// Return true and the result register by reference if it is possible.
+bool X86FastISel::X86FastEmitLoad(EVT VT, X86AddressMode &AM,
+ MachineMemOperand *MMO, unsigned &ResultReg,
+ unsigned Alignment) {
+ // Get opcode and regclass of the output for the given load instruction.
+ unsigned Opc = 0;
+ const TargetRegisterClass *RC = nullptr;
+ switch (VT.getSimpleVT().SimpleTy) {
+ default: return false;
+ case MVT::i1:
+ case MVT::i8:
+ Opc = X86::MOV8rm;
+ RC = &X86::GR8RegClass;
+ break;
+ case MVT::i16:
+ Opc = X86::MOV16rm;
+ RC = &X86::GR16RegClass;
+ break;
+ case MVT::i32:
+ Opc = X86::MOV32rm;
+ RC = &X86::GR32RegClass;
+ break;
+ case MVT::i64:
+ // Must be in x86-64 mode.
+ Opc = X86::MOV64rm;
+ RC = &X86::GR64RegClass;
+ break;
+ case MVT::f32:
+ if (X86ScalarSSEf32) {
+ Opc = Subtarget->hasAVX() ? X86::VMOVSSrm : X86::MOVSSrm;
+ RC = &X86::FR32RegClass;
+ } else {
+ Opc = X86::LD_Fp32m;
+ RC = &X86::RFP32RegClass;
+ }
+ break;
+ case MVT::f64:
+ if (X86ScalarSSEf64) {
+ Opc = Subtarget->hasAVX() ? X86::VMOVSDrm : X86::MOVSDrm;
+ RC = &X86::FR64RegClass;
+ } else {
+ Opc = X86::LD_Fp64m;
+ RC = &X86::RFP64RegClass;
+ }
+ break;
+ case MVT::f80:
+ // No f80 support yet.
+ return false;
+ case MVT::v4f32:
+ if (Alignment >= 16)
+ Opc = Subtarget->hasAVX() ? X86::VMOVAPSrm : X86::MOVAPSrm;
+ else
+ Opc = Subtarget->hasAVX() ? X86::VMOVUPSrm : X86::MOVUPSrm;
+ RC = &X86::VR128RegClass;
+ break;
+ case MVT::v2f64:
+ if (Alignment >= 16)
+ Opc = Subtarget->hasAVX() ? X86::VMOVAPDrm : X86::MOVAPDrm;
+ else
+ Opc = Subtarget->hasAVX() ? X86::VMOVUPDrm : X86::MOVUPDrm;
+ RC = &X86::VR128RegClass;
+ break;
+ case MVT::v4i32:
+ case MVT::v2i64:
+ case MVT::v8i16:
+ case MVT::v16i8:
+ if (Alignment >= 16)
+ Opc = Subtarget->hasAVX() ? X86::VMOVDQArm : X86::MOVDQArm;
+ else
+ Opc = Subtarget->hasAVX() ? X86::VMOVDQUrm : X86::MOVDQUrm;
+ RC = &X86::VR128RegClass;
+ break;
+ }
+
+ ResultReg = createResultReg(RC);
+ MachineInstrBuilder MIB =
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg);
+ addFullAddress(MIB, AM);
+ if (MMO)
+ MIB->addMemOperand(*FuncInfo.MF, MMO);
+ return true;
+}
+
+/// X86FastEmitStore - Emit a machine instruction to store a value Val of
+/// type VT. The address is either pre-computed, consisted of a base ptr, Ptr
+/// and a displacement offset, or a GlobalAddress,
+/// i.e. V. Return true if it is possible.
+bool X86FastISel::X86FastEmitStore(EVT VT, unsigned ValReg, bool ValIsKill,
+ X86AddressMode &AM,
+ MachineMemOperand *MMO, bool Aligned) {
+ // Get opcode and regclass of the output for the given store instruction.
+ unsigned Opc = 0;
+ switch (VT.getSimpleVT().SimpleTy) {
+ case MVT::f80: // No f80 support yet.
+ default: return false;
+ case MVT::i1: {
+ // Mask out all but lowest bit.
+ unsigned AndResult = createResultReg(&X86::GR8RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(X86::AND8ri), AndResult)
+ .addReg(ValReg, getKillRegState(ValIsKill)).addImm(1);
+ ValReg = AndResult;
+ }
+ // FALLTHROUGH, handling i1 as i8.
+ case MVT::i8: Opc = X86::MOV8mr; break;
+ case MVT::i16: Opc = X86::MOV16mr; break;
+ case MVT::i32: Opc = X86::MOV32mr; break;
+ case MVT::i64: Opc = X86::MOV64mr; break; // Must be in x86-64 mode.
+ case MVT::f32:
+ Opc = X86ScalarSSEf32 ?
+ (Subtarget->hasAVX() ? X86::VMOVSSmr : X86::MOVSSmr) : X86::ST_Fp32m;
+ break;
+ case MVT::f64:
+ Opc = X86ScalarSSEf64 ?
+ (Subtarget->hasAVX() ? X86::VMOVSDmr : X86::MOVSDmr) : X86::ST_Fp64m;
+ break;
+ case MVT::v4f32:
+ if (Aligned)
+ Opc = Subtarget->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr;
+ else
+ Opc = Subtarget->hasAVX() ? X86::VMOVUPSmr : X86::MOVUPSmr;
+ break;
+ case MVT::v2f64:
+ if (Aligned)
+ Opc = Subtarget->hasAVX() ? X86::VMOVAPDmr : X86::MOVAPDmr;
+ else
+ Opc = Subtarget->hasAVX() ? X86::VMOVUPDmr : X86::MOVUPDmr;
+ break;
+ case MVT::v4i32:
+ case MVT::v2i64:
+ case MVT::v8i16:
+ case MVT::v16i8:
+ if (Aligned)
+ Opc = Subtarget->hasAVX() ? X86::VMOVDQAmr : X86::MOVDQAmr;
+ else
+ Opc = Subtarget->hasAVX() ? X86::VMOVDQUmr : X86::MOVDQUmr;
+ break;
+ }
+
+ MachineInstrBuilder MIB =
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc));
+ addFullAddress(MIB, AM).addReg(ValReg, getKillRegState(ValIsKill));
+ if (MMO)
+ MIB->addMemOperand(*FuncInfo.MF, MMO);
+
+ return true;
+}
+
+bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val,
+ X86AddressMode &AM,
+ MachineMemOperand *MMO, bool Aligned) {
+ // Handle 'null' like i32/i64 0.
+ if (isa<ConstantPointerNull>(Val))
+ Val = Constant::getNullValue(DL.getIntPtrType(Val->getContext()));
+
+ // If this is a store of a simple constant, fold the constant into the store.
+ if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val)) {
+ unsigned Opc = 0;
+ bool Signed = true;
+ switch (VT.getSimpleVT().SimpleTy) {
+ default: break;
+ case MVT::i1: Signed = false; // FALLTHROUGH to handle as i8.
+ case MVT::i8: Opc = X86::MOV8mi; break;
+ case MVT::i16: Opc = X86::MOV16mi; break;
+ case MVT::i32: Opc = X86::MOV32mi; break;
+ case MVT::i64:
+ // Must be a 32-bit sign extended value.
+ if (isInt<32>(CI->getSExtValue()))
+ Opc = X86::MOV64mi32;
+ break;
+ }
+
+ if (Opc) {
+ MachineInstrBuilder MIB =
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc));
+ addFullAddress(MIB, AM).addImm(Signed ? (uint64_t) CI->getSExtValue()
+ : CI->getZExtValue());
+ if (MMO)
+ MIB->addMemOperand(*FuncInfo.MF, MMO);
+ return true;
+ }
+ }
+
+ unsigned ValReg = getRegForValue(Val);
+ if (ValReg == 0)
+ return false;
+
+ bool ValKill = hasTrivialKill(Val);
+ return X86FastEmitStore(VT, ValReg, ValKill, AM, MMO, Aligned);
+}
+
+/// X86FastEmitExtend - Emit a machine instruction to extend a value Src of
+/// type SrcVT to type DstVT using the specified extension opcode Opc (e.g.
+/// ISD::SIGN_EXTEND).
+bool X86FastISel::X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT,
+ unsigned Src, EVT SrcVT,
+ unsigned &ResultReg) {
+ unsigned RR = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), Opc,
+ Src, /*TODO: Kill=*/false);
+ if (RR == 0)
+ return false;
+
+ ResultReg = RR;
+ return true;
+}
+
+bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
+ // Handle constant address.
+ if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
+ // Can't handle alternate code models yet.
+ if (TM.getCodeModel() != CodeModel::Small)
+ return false;
+
+ // Can't handle TLS yet.
+ if (GV->isThreadLocal())
+ return false;
+
+ // RIP-relative addresses can't have additional register operands, so if
+ // we've already folded stuff into the addressing mode, just force the
+ // global value into its own register, which we can use as the basereg.
+ if (!Subtarget->isPICStyleRIPRel() ||
+ (AM.Base.Reg == 0 && AM.IndexReg == 0)) {
+ // Okay, we've committed to selecting this global. Set up the address.
+ AM.GV = GV;
+
+ // Allow the subtarget to classify the global.
+ unsigned char GVFlags = Subtarget->ClassifyGlobalReference(GV, TM);
+
+ // If this reference is relative to the pic base, set it now.
+ if (isGlobalRelativeToPICBase(GVFlags)) {
+ // FIXME: How do we know Base.Reg is free??
+ AM.Base.Reg = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
+ }
+
+ // Unless the ABI requires an extra load, return a direct reference to
+ // the global.
+ if (!isGlobalStubReference(GVFlags)) {
+ if (Subtarget->isPICStyleRIPRel()) {
+ // Use rip-relative addressing if we can. Above we verified that the
+ // base and index registers are unused.
+ assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
+ AM.Base.Reg = X86::RIP;
+ }
+ AM.GVOpFlags = GVFlags;
+ return true;
+ }
+
+ // Ok, we need to do a load from a stub. If we've already loaded from
+ // this stub, reuse the loaded pointer, otherwise emit the load now.
+ DenseMap<const Value *, unsigned>::iterator I = LocalValueMap.find(V);
+ unsigned LoadReg;
+ if (I != LocalValueMap.end() && I->second != 0) {
+ LoadReg = I->second;
+ } else {
+ // Issue load from stub.
+ unsigned Opc = 0;
+ const TargetRegisterClass *RC = nullptr;
+ X86AddressMode StubAM;
+ StubAM.Base.Reg = AM.Base.Reg;
+ StubAM.GV = GV;
+ StubAM.GVOpFlags = GVFlags;
+
+ // Prepare for inserting code in the local-value area.
+ SavePoint SaveInsertPt = enterLocalValueArea();
+
+ if (TLI.getPointerTy(DL) == MVT::i64) {
+ Opc = X86::MOV64rm;
+ RC = &X86::GR64RegClass;
+
+ if (Subtarget->isPICStyleRIPRel())
+ StubAM.Base.Reg = X86::RIP;
+ } else {
+ Opc = X86::MOV32rm;
+ RC = &X86::GR32RegClass;
+ }
+
+ LoadReg = createResultReg(RC);
+ MachineInstrBuilder LoadMI =
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), LoadReg);
+ addFullAddress(LoadMI, StubAM);
+
+ // Ok, back to normal mode.
+ leaveLocalValueArea(SaveInsertPt);
+
+ // Prevent loading GV stub multiple times in same MBB.
+ LocalValueMap[V] = LoadReg;
+ }
+
+ // Now construct the final address. Note that the Disp, Scale,
+ // and Index values may already be set here.
+ AM.Base.Reg = LoadReg;
+ AM.GV = nullptr;
+ return true;
+ }
+ }
+
+ // If all else fails, try to materialize the value in a register.
+ if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
+ if (AM.Base.Reg == 0) {
+ AM.Base.Reg = getRegForValue(V);
+ return AM.Base.Reg != 0;
+ }
+ if (AM.IndexReg == 0) {
+ assert(AM.Scale == 1 && "Scale with no index!");
+ AM.IndexReg = getRegForValue(V);
+ return AM.IndexReg != 0;
+ }
+ }
+
+ return false;
+}
+
+/// X86SelectAddress - Attempt to fill in an address from the given value.
+///
+bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
+ SmallVector<const Value *, 32> GEPs;
+redo_gep:
+ const User *U = nullptr;
+ unsigned Opcode = Instruction::UserOp1;
+ if (const Instruction *I = dyn_cast<Instruction>(V)) {
+ // Don't walk into other basic blocks; it's possible we haven't
+ // visited them yet, so the instructions may not yet be assigned
+ // virtual registers.
+ if (FuncInfo.StaticAllocaMap.count(static_cast<const AllocaInst *>(V)) ||
+ FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) {
+ Opcode = I->getOpcode();
+ U = I;
+ }
+ } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(V)) {
+ Opcode = C->getOpcode();
+ U = C;
+ }
+
+ if (PointerType *Ty = dyn_cast<PointerType>(V->getType()))
+ if (Ty->getAddressSpace() > 255)
+ // Fast instruction selection doesn't support the special
+ // address spaces.
+ return false;
+
+ switch (Opcode) {
+ default: break;
+ case Instruction::BitCast:
+ // Look past bitcasts.
+ return X86SelectAddress(U->getOperand(0), AM);
+
+ case Instruction::IntToPtr:
+ // Look past no-op inttoptrs.
+ if (TLI.getValueType(DL, U->getOperand(0)->getType()) ==
+ TLI.getPointerTy(DL))
+ return X86SelectAddress(U->getOperand(0), AM);
+ break;
+
+ case Instruction::PtrToInt:
+ // Look past no-op ptrtoints.
+ if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
+ return X86SelectAddress(U->getOperand(0), AM);
+ break;
+
+ case Instruction::Alloca: {
+ // Do static allocas.
+ const AllocaInst *A = cast<AllocaInst>(V);
+ DenseMap<const AllocaInst *, int>::iterator SI =
+ FuncInfo.StaticAllocaMap.find(A);
+ if (SI != FuncInfo.StaticAllocaMap.end()) {
+ AM.BaseType = X86AddressMode::FrameIndexBase;
+ AM.Base.FrameIndex = SI->second;
+ return true;
+ }
+ break;
+ }
+
+ case Instruction::Add: {
+ // Adds of constants are common and easy enough.
+ if (const ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
+ uint64_t Disp = (int32_t)AM.Disp + (uint64_t)CI->getSExtValue();
+ // They have to fit in the 32-bit signed displacement field though.
+ if (isInt<32>(Disp)) {
+ AM.Disp = (uint32_t)Disp;
+ return X86SelectAddress(U->getOperand(0), AM);
+ }
+ }
+ break;
+ }
+
+ case Instruction::GetElementPtr: {
+ X86AddressMode SavedAM = AM;
+
+ // Pattern-match simple GEPs.
+ uint64_t Disp = (int32_t)AM.Disp;
+ unsigned IndexReg = AM.IndexReg;
+ unsigned Scale = AM.Scale;
+ gep_type_iterator GTI = gep_type_begin(U);
+ // Iterate through the indices, folding what we can. Constants can be
+ // folded, and one dynamic index can be handled, if the scale is supported.
+ for (User::const_op_iterator i = U->op_begin() + 1, e = U->op_end();
+ i != e; ++i, ++GTI) {
+ const Value *Op = *i;
+ if (StructType *STy = dyn_cast<StructType>(*GTI)) {
+ const StructLayout *SL = DL.getStructLayout(STy);
+ Disp += SL->getElementOffset(cast<ConstantInt>(Op)->getZExtValue());
+ continue;
+ }
+
+ // A array/variable index is always of the form i*S where S is the
+ // constant scale size. See if we can push the scale into immediates.
+ uint64_t S = DL.getTypeAllocSize(GTI.getIndexedType());
+ for (;;) {
+ if (const ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {
+ // Constant-offset addressing.
+ Disp += CI->getSExtValue() * S;
+ break;
+ }
+ if (canFoldAddIntoGEP(U, Op)) {
+ // A compatible add with a constant operand. Fold the constant.
+ ConstantInt *CI =
+ cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1));
+ Disp += CI->getSExtValue() * S;
+ // Iterate on the other operand.
+ Op = cast<AddOperator>(Op)->getOperand(0);
+ continue;
+ }
+ if (IndexReg == 0 &&
+ (!AM.GV || !Subtarget->isPICStyleRIPRel()) &&
+ (S == 1 || S == 2 || S == 4 || S == 8)) {
+ // Scaled-index addressing.
+ Scale = S;
+ IndexReg = getRegForGEPIndex(Op).first;
+ if (IndexReg == 0)
+ return false;
+ break;
+ }
+ // Unsupported.
+ goto unsupported_gep;
+ }
+ }
+
+ // Check for displacement overflow.
+ if (!isInt<32>(Disp))
+ break;
+
+ AM.IndexReg = IndexReg;
+ AM.Scale = Scale;
+ AM.Disp = (uint32_t)Disp;
+ GEPs.push_back(V);
+
+ if (const GetElementPtrInst *GEP =
+ dyn_cast<GetElementPtrInst>(U->getOperand(0))) {
+ // Ok, the GEP indices were covered by constant-offset and scaled-index
+ // addressing. Update the address state and move on to examining the base.
+ V = GEP;
+ goto redo_gep;
+ } else if (X86SelectAddress(U->getOperand(0), AM)) {
+ return true;
+ }
+
+ // If we couldn't merge the gep value into this addr mode, revert back to
+ // our address and just match the value instead of completely failing.
+ AM = SavedAM;
+
+ for (SmallVectorImpl<const Value *>::reverse_iterator
+ I = GEPs.rbegin(), E = GEPs.rend(); I != E; ++I)
+ if (handleConstantAddresses(*I, AM))
+ return true;
+
+ return false;
+ unsupported_gep:
+ // Ok, the GEP indices weren't all covered.
+ break;
+ }
+ }
+
+ return handleConstantAddresses(V, AM);
+}
+
+/// X86SelectCallAddress - Attempt to fill in an address from the given value.
+///
+bool X86FastISel::X86SelectCallAddress(const Value *V, X86AddressMode &AM) {
+ const User *U = nullptr;
+ unsigned Opcode = Instruction::UserOp1;
+ const Instruction *I = dyn_cast<Instruction>(V);
+ // Record if the value is defined in the same basic block.
+ //
+ // This information is crucial to know whether or not folding an
+ // operand is valid.
+ // Indeed, FastISel generates or reuses a virtual register for all
+ // operands of all instructions it selects. Obviously, the definition and
+ // its uses must use the same virtual register otherwise the produced
+ // code is incorrect.
+ // Before instruction selection, FunctionLoweringInfo::set sets the virtual
+ // registers for values that are alive across basic blocks. This ensures
+ // that the values are consistently set between across basic block, even
+ // if different instruction selection mechanisms are used (e.g., a mix of
+ // SDISel and FastISel).
+ // For values local to a basic block, the instruction selection process
+ // generates these virtual registers with whatever method is appropriate
+ // for its needs. In particular, FastISel and SDISel do not share the way
+ // local virtual registers are set.
+ // Therefore, this is impossible (or at least unsafe) to share values
+ // between basic blocks unless they use the same instruction selection
+ // method, which is not guarantee for X86.
+ // Moreover, things like hasOneUse could not be used accurately, if we
+ // allow to reference values across basic blocks whereas they are not
+ // alive across basic blocks initially.
+ bool InMBB = true;
+ if (I) {
+ Opcode = I->getOpcode();
+ U = I;
+ InMBB = I->getParent() == FuncInfo.MBB->getBasicBlock();
+ } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(V)) {
+ Opcode = C->getOpcode();
+ U = C;
+ }
+
+ switch (Opcode) {
+ default: break;
+ case Instruction::BitCast:
+ // Look past bitcasts if its operand is in the same BB.
+ if (InMBB)
+ return X86SelectCallAddress(U->getOperand(0), AM);
+ break;
+
+ case Instruction::IntToPtr:
+ // Look past no-op inttoptrs if its operand is in the same BB.
+ if (InMBB &&
+ TLI.getValueType(DL, U->getOperand(0)->getType()) ==
+ TLI.getPointerTy(DL))
+ return X86SelectCallAddress(U->getOperand(0), AM);
+ break;
+
+ case Instruction::PtrToInt:
+ // Look past no-op ptrtoints if its operand is in the same BB.
+ if (InMBB && TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
+ return X86SelectCallAddress(U->getOperand(0), AM);
+ break;
+ }
+
+ // Handle constant address.
+ if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
+ // Can't handle alternate code models yet.
+ if (TM.getCodeModel() != CodeModel::Small)
+ return false;
+
+ // RIP-relative addresses can't have additional register operands.
+ if (Subtarget->isPICStyleRIPRel() &&
+ (AM.Base.Reg != 0 || AM.IndexReg != 0))
+ return false;
+
+ // Can't handle DLL Import.
+ if (GV->hasDLLImportStorageClass())
+ return false;
+
+ // Can't handle TLS.
+ if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV))
+ if (GVar->isThreadLocal())
+ return false;
+
+ // Okay, we've committed to selecting this global. Set up the basic address.
+ AM.GV = GV;
+
+ // No ABI requires an extra load for anything other than DLLImport, which
+ // we rejected above. Return a direct reference to the global.
+ if (Subtarget->isPICStyleRIPRel()) {
+ // Use rip-relative addressing if we can. Above we verified that the
+ // base and index registers are unused.
+ assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
+ AM.Base.Reg = X86::RIP;
+ } else if (Subtarget->isPICStyleStubPIC()) {
+ AM.GVOpFlags = X86II::MO_PIC_BASE_OFFSET;
+ } else if (Subtarget->isPICStyleGOT()) {
+ AM.GVOpFlags = X86II::MO_GOTOFF;
+ }
+
+ return true;
+ }
+
+ // If all else fails, try to materialize the value in a register.
+ if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
+ if (AM.Base.Reg == 0) {
+ AM.Base.Reg = getRegForValue(V);
+ return AM.Base.Reg != 0;
+ }
+ if (AM.IndexReg == 0) {
+ assert(AM.Scale == 1 && "Scale with no index!");
+ AM.IndexReg = getRegForValue(V);
+ return AM.IndexReg != 0;
+ }
+ }
+
+ return false;
+}
+
+
+/// X86SelectStore - Select and emit code to implement store instructions.
+bool X86FastISel::X86SelectStore(const Instruction *I) {
+ // Atomic stores need special handling.
+ const StoreInst *S = cast<StoreInst>(I);
+
+ if (S->isAtomic())
+ return false;
+
+ const Value *Val = S->getValueOperand();
+ const Value *Ptr = S->getPointerOperand();
+
+ MVT VT;
+ if (!isTypeLegal(Val->getType(), VT, /*AllowI1=*/true))
+ return false;
+
+ unsigned Alignment = S->getAlignment();
+ unsigned ABIAlignment = DL.getABITypeAlignment(Val->getType());
+ if (Alignment == 0) // Ensure that codegen never sees alignment 0
+ Alignment = ABIAlignment;
+ bool Aligned = Alignment >= ABIAlignment;
+
+ X86AddressMode AM;
+ if (!X86SelectAddress(Ptr, AM))
+ return false;
+
+ return X86FastEmitStore(VT, Val, AM, createMachineMemOperandFor(I), Aligned);
+}
+
+/// X86SelectRet - Select and emit code to implement ret instructions.
+bool X86FastISel::X86SelectRet(const Instruction *I) {
+ const ReturnInst *Ret = cast<ReturnInst>(I);
+ const Function &F = *I->getParent()->getParent();
+ const X86MachineFunctionInfo *X86MFInfo =
+ FuncInfo.MF->getInfo<X86MachineFunctionInfo>();
+
+ if (!FuncInfo.CanLowerReturn)
+ return false;
+
+ CallingConv::ID CC = F.getCallingConv();
+ if (CC != CallingConv::C &&
+ CC != CallingConv::Fast &&
+ CC != CallingConv::X86_FastCall &&
+ CC != CallingConv::X86_64_SysV)
+ return false;
+
+ if (Subtarget->isCallingConvWin64(CC))
+ return false;
+
+ // Don't handle popping bytes on return for now.
+ if (X86MFInfo->getBytesToPopOnReturn() != 0)
+ return false;
+
+ // fastcc with -tailcallopt is intended to provide a guaranteed
+ // tail call optimization. Fastisel doesn't know how to do that.
+ if (CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt)
+ return false;
+
+ // Let SDISel handle vararg functions.
+ if (F.isVarArg())
+ return false;
+
+ // Build a list of return value registers.
+ SmallVector<unsigned, 4> RetRegs;
+
+ if (Ret->getNumOperands() > 0) {
+ SmallVector<ISD::OutputArg, 4> Outs;
+ GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI, DL);
+
+ // Analyze operands of the call, assigning locations to each operand.
+ SmallVector<CCValAssign, 16> ValLocs;
+ CCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, ValLocs, I->getContext());
+ CCInfo.AnalyzeReturn(Outs, RetCC_X86);
+
+ const Value *RV = Ret->getOperand(0);
+ unsigned Reg = getRegForValue(RV);
+ if (Reg == 0)
+ return false;
+
+ // Only handle a single return value for now.
+ if (ValLocs.size() != 1)
+ return false;
+
+ CCValAssign &VA = ValLocs[0];
+
+ // Don't bother handling odd stuff for now.
+ if (VA.getLocInfo() != CCValAssign::Full)
+ return false;
+ // Only handle register returns for now.
+ if (!VA.isRegLoc())
+ return false;
+
+ // The calling-convention tables for x87 returns don't tell
+ // the whole story.
+ if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)
+ return false;
+
+ unsigned SrcReg = Reg + VA.getValNo();
+ EVT SrcVT = TLI.getValueType(DL, RV->getType());
+ EVT DstVT = VA.getValVT();
+ // Special handling for extended integers.
+ if (SrcVT != DstVT) {
+ if (SrcVT != MVT::i1 && SrcVT != MVT::i8 && SrcVT != MVT::i16)
+ return false;
+
+ if (!Outs[0].Flags.isZExt() && !Outs[0].Flags.isSExt())
+ return false;
+
+ assert(DstVT == MVT::i32 && "X86 should always ext to i32");
+
+ if (SrcVT == MVT::i1) {
+ if (Outs[0].Flags.isSExt())
+ return false;
+ SrcReg = fastEmitZExtFromI1(MVT::i8, SrcReg, /*TODO: Kill=*/false);
+ SrcVT = MVT::i8;
+ }
+ unsigned Op = Outs[0].Flags.isZExt() ? ISD::ZERO_EXTEND :
+ ISD::SIGN_EXTEND;
+ SrcReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), Op,
+ SrcReg, /*TODO: Kill=*/false);
+ }
+
+ // Make the copy.
+ unsigned DstReg = VA.getLocReg();
+ const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
+ // Avoid a cross-class copy. This is very unlikely.
+ if (!SrcRC->contains(DstReg))
+ return false;
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), DstReg).addReg(SrcReg);
+
+ // Add register to return instruction.
+ RetRegs.push_back(VA.getLocReg());
+ }
+
+ // The x86-64 ABI for returning structs by value requires that we copy
+ // the sret argument into %rax for the return. We saved the argument into
+ // a virtual register in the entry block, so now we copy the value out
+ // and into %rax. We also do the same with %eax for Win32.
+ if (F.hasStructRetAttr() &&
+ (Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC())) {
+ unsigned Reg = X86MFInfo->getSRetReturnReg();
+ assert(Reg &&
+ "SRetReturnReg should have been set in LowerFormalArguments()!");
+ unsigned RetReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), RetReg).addReg(Reg);
+ RetRegs.push_back(RetReg);
+ }
+
+ // Now emit the RET.
+ MachineInstrBuilder MIB =
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Subtarget->is64Bit() ? X86::RETQ : X86::RETL));
+ for (unsigned i = 0, e = RetRegs.size(); i != e; ++i)
+ MIB.addReg(RetRegs[i], RegState::Implicit);
+ return true;
+}
+
+/// X86SelectLoad - Select and emit code to implement load instructions.
+///
+bool X86FastISel::X86SelectLoad(const Instruction *I) {
+ const LoadInst *LI = cast<LoadInst>(I);
+
+ // Atomic loads need special handling.
+ if (LI->isAtomic())
+ return false;
+
+ MVT VT;
+ if (!isTypeLegal(LI->getType(), VT, /*AllowI1=*/true))
+ return false;
+
+ const Value *Ptr = LI->getPointerOperand();
+
+ X86AddressMode AM;
+ if (!X86SelectAddress(Ptr, AM))
+ return false;
+
+ unsigned Alignment = LI->getAlignment();
+ unsigned ABIAlignment = DL.getABITypeAlignment(LI->getType());
+ if (Alignment == 0) // Ensure that codegen never sees alignment 0
+ Alignment = ABIAlignment;
+
+ unsigned ResultReg = 0;
+ if (!X86FastEmitLoad(VT, AM, createMachineMemOperandFor(LI), ResultReg,
+ Alignment))
+ return false;
+
+ updateValueMap(I, ResultReg);
+ return true;
+}
+
+static unsigned X86ChooseCmpOpcode(EVT VT, const X86Subtarget *Subtarget) {
+ bool HasAVX = Subtarget->hasAVX();
+ bool X86ScalarSSEf32 = Subtarget->hasSSE1();
+ bool X86ScalarSSEf64 = Subtarget->hasSSE2();
+
+ switch (VT.getSimpleVT().SimpleTy) {
+ default: return 0;
+ case MVT::i8: return X86::CMP8rr;
+ case MVT::i16: return X86::CMP16rr;
+ case MVT::i32: return X86::CMP32rr;
+ case MVT::i64: return X86::CMP64rr;
+ case MVT::f32:
+ return X86ScalarSSEf32 ? (HasAVX ? X86::VUCOMISSrr : X86::UCOMISSrr) : 0;
+ case MVT::f64:
+ return X86ScalarSSEf64 ? (HasAVX ? X86::VUCOMISDrr : X86::UCOMISDrr) : 0;
+ }
+}
+
+/// If we have a comparison with RHS as the RHS of the comparison, return an
+/// opcode that works for the compare (e.g. CMP32ri) otherwise return 0.
+static unsigned X86ChooseCmpImmediateOpcode(EVT VT, const ConstantInt *RHSC) {
+ int64_t Val = RHSC->getSExtValue();
+ switch (VT.getSimpleVT().SimpleTy) {
+ // Otherwise, we can't fold the immediate into this comparison.
+ default:
+ return 0;
+ case MVT::i8:
+ return X86::CMP8ri;
+ case MVT::i16:
+ if (isInt<8>(Val))
+ return X86::CMP16ri8;
+ return X86::CMP16ri;
+ case MVT::i32:
+ if (isInt<8>(Val))
+ return X86::CMP32ri8;
+ return X86::CMP32ri;
+ case MVT::i64:
+ if (isInt<8>(Val))
+ return X86::CMP64ri8;
+ // 64-bit comparisons are only valid if the immediate fits in a 32-bit sext
+ // field.
+ if (isInt<32>(Val))
+ return X86::CMP64ri32;
+ return 0;
+ }
+}
+
+bool X86FastISel::X86FastEmitCompare(const Value *Op0, const Value *Op1,
+ EVT VT, DebugLoc CurDbgLoc) {
+ unsigned Op0Reg = getRegForValue(Op0);
+ if (Op0Reg == 0) return false;
+
+ // Handle 'null' like i32/i64 0.
+ if (isa<ConstantPointerNull>(Op1))
+ Op1 = Constant::getNullValue(DL.getIntPtrType(Op0->getContext()));
+
+ // We have two options: compare with register or immediate. If the RHS of
+ // the compare is an immediate that we can fold into this compare, use
+ // CMPri, otherwise use CMPrr.
+ if (const ConstantInt *Op1C = dyn_cast<ConstantInt>(Op1)) {
+ if (unsigned CompareImmOpc = X86ChooseCmpImmediateOpcode(VT, Op1C)) {
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurDbgLoc, TII.get(CompareImmOpc))
+ .addReg(Op0Reg)
+ .addImm(Op1C->getSExtValue());
+ return true;
+ }
+ }
+
+ unsigned CompareOpc = X86ChooseCmpOpcode(VT, Subtarget);
+ if (CompareOpc == 0) return false;
+
+ unsigned Op1Reg = getRegForValue(Op1);
+ if (Op1Reg == 0) return false;
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurDbgLoc, TII.get(CompareOpc))
+ .addReg(Op0Reg)
+ .addReg(Op1Reg);
+
+ return true;
+}
+
+bool X86FastISel::X86SelectCmp(const Instruction *I) {
+ const CmpInst *CI = cast<CmpInst>(I);
+
+ MVT VT;
+ if (!isTypeLegal(I->getOperand(0)->getType(), VT))
+ return false;
+
+ // Try to optimize or fold the cmp.
+ CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
+ unsigned ResultReg = 0;
+ switch (Predicate) {
+ default: break;
+ case CmpInst::FCMP_FALSE: {
+ ResultReg = createResultReg(&X86::GR32RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV32r0),
+ ResultReg);
+ ResultReg = fastEmitInst_extractsubreg(MVT::i8, ResultReg, /*Kill=*/true,
+ X86::sub_8bit);
+ if (!ResultReg)
+ return false;
+ break;
+ }
+ case CmpInst::FCMP_TRUE: {
+ ResultReg = createResultReg(&X86::GR8RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV8ri),
+ ResultReg).addImm(1);
+ break;
+ }
+ }
+
+ if (ResultReg) {
+ updateValueMap(I, ResultReg);
+ return true;
+ }
+
+ const Value *LHS = CI->getOperand(0);
+ const Value *RHS = CI->getOperand(1);
+
+ // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
+ // We don't have to materialize a zero constant for this case and can just use
+ // %x again on the RHS.
+ if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
+ const auto *RHSC = dyn_cast<ConstantFP>(RHS);
+ if (RHSC && RHSC->isNullValue())
+ RHS = LHS;
+ }
+
+ // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
+ static unsigned SETFOpcTable[2][3] = {
+ { X86::SETEr, X86::SETNPr, X86::AND8rr },
+ { X86::SETNEr, X86::SETPr, X86::OR8rr }
+ };
+ unsigned *SETFOpc = nullptr;
+ switch (Predicate) {
+ default: break;
+ case CmpInst::FCMP_OEQ: SETFOpc = &SETFOpcTable[0][0]; break;
+ case CmpInst::FCMP_UNE: SETFOpc = &SETFOpcTable[1][0]; break;
+ }
+
+ ResultReg = createResultReg(&X86::GR8RegClass);
+ if (SETFOpc) {
+ if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc()))
+ return false;
+
+ unsigned FlagReg1 = createResultReg(&X86::GR8RegClass);
+ unsigned FlagReg2 = createResultReg(&X86::GR8RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[0]),
+ FlagReg1);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[1]),
+ FlagReg2);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[2]),
+ ResultReg).addReg(FlagReg1).addReg(FlagReg2);
+ updateValueMap(I, ResultReg);
+ return true;
+ }
+
+ X86::CondCode CC;
+ bool SwapArgs;
+ std::tie(CC, SwapArgs) = getX86ConditionCode(Predicate);
+ assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
+ unsigned Opc = X86::getSETFromCond(CC);
+
+ if (SwapArgs)
+ std::swap(LHS, RHS);
+
+ // Emit a compare of LHS/RHS.
+ if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc()))
+ return false;
+
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg);
+ updateValueMap(I, ResultReg);
+ return true;
+}
+
+bool X86FastISel::X86SelectZExt(const Instruction *I) {
+ EVT DstVT = TLI.getValueType(DL, I->getType());
+ if (!TLI.isTypeLegal(DstVT))
+ return false;
+
+ unsigned ResultReg = getRegForValue(I->getOperand(0));
+ if (ResultReg == 0)
+ return false;
+
+ // Handle zero-extension from i1 to i8, which is common.
+ MVT SrcVT = TLI.getSimpleValueType(DL, I->getOperand(0)->getType());
+ if (SrcVT.SimpleTy == MVT::i1) {
+ // Set the high bits to zero.
+ ResultReg = fastEmitZExtFromI1(MVT::i8, ResultReg, /*TODO: Kill=*/false);
+ SrcVT = MVT::i8;
+
+ if (ResultReg == 0)
+ return false;
+ }
+
+ if (DstVT == MVT::i64) {
+ // Handle extension to 64-bits via sub-register shenanigans.
+ unsigned MovInst;
+
+ switch (SrcVT.SimpleTy) {
+ case MVT::i8: MovInst = X86::MOVZX32rr8; break;
+ case MVT::i16: MovInst = X86::MOVZX32rr16; break;
+ case MVT::i32: MovInst = X86::MOV32rr; break;
+ default: llvm_unreachable("Unexpected zext to i64 source type");
+ }
+
+ unsigned Result32 = createResultReg(&X86::GR32RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(MovInst), Result32)
+ .addReg(ResultReg);
+
+ ResultReg = createResultReg(&X86::GR64RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::SUBREG_TO_REG),
+ ResultReg)
+ .addImm(0).addReg(Result32).addImm(X86::sub_32bit);
+ } else if (DstVT != MVT::i8) {
+ ResultReg = fastEmit_r(MVT::i8, DstVT.getSimpleVT(), ISD::ZERO_EXTEND,
+ ResultReg, /*Kill=*/true);
+ if (ResultReg == 0)
+ return false;
+ }
+
+ updateValueMap(I, ResultReg);
+ return true;
+}
+
+bool X86FastISel::X86SelectBranch(const Instruction *I) {
+ // Unconditional branches are selected by tablegen-generated code.
+ // Handle a conditional branch.
+ const BranchInst *BI = cast<BranchInst>(I);
+ MachineBasicBlock *TrueMBB = FuncInfo.MBBMap[BI->getSuccessor(0)];
+ MachineBasicBlock *FalseMBB = FuncInfo.MBBMap[BI->getSuccessor(1)];
+
+ // Fold the common case of a conditional branch with a comparison
+ // in the same block (values defined on other blocks may not have
+ // initialized registers).
+ X86::CondCode CC;
+ if (const CmpInst *CI = dyn_cast<CmpInst>(BI->getCondition())) {
+ if (CI->hasOneUse() && CI->getParent() == I->getParent()) {
+ EVT VT = TLI.getValueType(DL, CI->getOperand(0)->getType());
+
+ // Try to optimize or fold the cmp.
+ CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
+ switch (Predicate) {
+ default: break;
+ case CmpInst::FCMP_FALSE: fastEmitBranch(FalseMBB, DbgLoc); return true;
+ case CmpInst::FCMP_TRUE: fastEmitBranch(TrueMBB, DbgLoc); return true;
+ }
+
+ const Value *CmpLHS = CI->getOperand(0);
+ const Value *CmpRHS = CI->getOperand(1);
+
+ // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x,
+ // 0.0.
+ // We don't have to materialize a zero constant for this case and can just
+ // use %x again on the RHS.
+ if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
+ const auto *CmpRHSC = dyn_cast<ConstantFP>(CmpRHS);
+ if (CmpRHSC && CmpRHSC->isNullValue())
+ CmpRHS = CmpLHS;
+ }
+
+ // Try to take advantage of fallthrough opportunities.
+ if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {
+ std::swap(TrueMBB, FalseMBB);
+ Predicate = CmpInst::getInversePredicate(Predicate);
+ }
+
+ // FCMP_OEQ and FCMP_UNE cannot be expressed with a single flag/condition
+ // code check. Instead two branch instructions are required to check all
+ // the flags. First we change the predicate to a supported condition code,
+ // which will be the first branch. Later one we will emit the second
+ // branch.
+ bool NeedExtraBranch = false;
+ switch (Predicate) {
+ default: break;
+ case CmpInst::FCMP_OEQ:
+ std::swap(TrueMBB, FalseMBB); // fall-through
+ case CmpInst::FCMP_UNE:
+ NeedExtraBranch = true;
+ Predicate = CmpInst::FCMP_ONE;
+ break;
+ }
+
+ bool SwapArgs;
+ unsigned BranchOpc;
+ std::tie(CC, SwapArgs) = getX86ConditionCode(Predicate);
+ assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
+
+ BranchOpc = X86::GetCondBranchFromCond(CC);
+ if (SwapArgs)
+ std::swap(CmpLHS, CmpRHS);
+
+ // Emit a compare of the LHS and RHS, setting the flags.
+ if (!X86FastEmitCompare(CmpLHS, CmpRHS, VT, CI->getDebugLoc()))
+ return false;
+
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(BranchOpc))
+ .addMBB(TrueMBB);
+
+ // X86 requires a second branch to handle UNE (and OEQ, which is mapped
+ // to UNE above).
+ if (NeedExtraBranch) {
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::JP_1))
+ .addMBB(TrueMBB);
+ }
+
+ finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
+ return true;
+ }
+ } else if (TruncInst *TI = dyn_cast<TruncInst>(BI->getCondition())) {
+ // Handle things like "%cond = trunc i32 %X to i1 / br i1 %cond", which
+ // typically happen for _Bool and C++ bools.
+ MVT SourceVT;
+ if (TI->hasOneUse() && TI->getParent() == I->getParent() &&
+ isTypeLegal(TI->getOperand(0)->getType(), SourceVT)) {
+ unsigned TestOpc = 0;
+ switch (SourceVT.SimpleTy) {
+ default: break;
+ case MVT::i8: TestOpc = X86::TEST8ri; break;
+ case MVT::i16: TestOpc = X86::TEST16ri; break;
+ case MVT::i32: TestOpc = X86::TEST32ri; break;
+ case MVT::i64: TestOpc = X86::TEST64ri32; break;
+ }
+ if (TestOpc) {
+ unsigned OpReg = getRegForValue(TI->getOperand(0));
+ if (OpReg == 0) return false;
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TestOpc))
+ .addReg(OpReg).addImm(1);
+
+ unsigned JmpOpc = X86::JNE_1;
+ if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {
+ std::swap(TrueMBB, FalseMBB);
+ JmpOpc = X86::JE_1;
+ }
+
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(JmpOpc))
+ .addMBB(TrueMBB);
+
+ finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
+ return true;
+ }
+ }
+ } else if (foldX86XALUIntrinsic(CC, BI, BI->getCondition())) {
+ // Fake request the condition, otherwise the intrinsic might be completely
+ // optimized away.
+ unsigned TmpReg = getRegForValue(BI->getCondition());
+ if (TmpReg == 0)
+ return false;
+
+ unsigned BranchOpc = X86::GetCondBranchFromCond(CC);
+
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(BranchOpc))
+ .addMBB(TrueMBB);
+ finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
+ return true;
+ }
+
+ // Otherwise do a clumsy setcc and re-test it.
+ // Note that i1 essentially gets ANY_EXTEND'ed to i8 where it isn't used
+ // in an explicit cast, so make sure to handle that correctly.
+ unsigned OpReg = getRegForValue(BI->getCondition());
+ if (OpReg == 0) return false;
+
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TEST8ri))
+ .addReg(OpReg).addImm(1);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::JNE_1))
+ .addMBB(TrueMBB);
+ finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
+ return true;
+}
+
+bool X86FastISel::X86SelectShift(const Instruction *I) {
+ unsigned CReg = 0, OpReg = 0;
+ const TargetRegisterClass *RC = nullptr;
+ if (I->getType()->isIntegerTy(8)) {
+ CReg = X86::CL;
+ RC = &X86::GR8RegClass;
+ switch (I->getOpcode()) {
+ case Instruction::LShr: OpReg = X86::SHR8rCL; break;
+ case Instruction::AShr: OpReg = X86::SAR8rCL; break;
+ case Instruction::Shl: OpReg = X86::SHL8rCL; break;
+ default: return false;
+ }
+ } else if (I->getType()->isIntegerTy(16)) {
+ CReg = X86::CX;
+ RC = &X86::GR16RegClass;
+ switch (I->getOpcode()) {
+ case Instruction::LShr: OpReg = X86::SHR16rCL; break;
+ case Instruction::AShr: OpReg = X86::SAR16rCL; break;
+ case Instruction::Shl: OpReg = X86::SHL16rCL; break;
+ default: return false;
+ }
+ } else if (I->getType()->isIntegerTy(32)) {
+ CReg = X86::ECX;
+ RC = &X86::GR32RegClass;
+ switch (I->getOpcode()) {
+ case Instruction::LShr: OpReg = X86::SHR32rCL; break;
+ case Instruction::AShr: OpReg = X86::SAR32rCL; break;
+ case Instruction::Shl: OpReg = X86::SHL32rCL; break;
+ default: return false;
+ }
+ } else if (I->getType()->isIntegerTy(64)) {
+ CReg = X86::RCX;
+ RC = &X86::GR64RegClass;
+ switch (I->getOpcode()) {
+ case Instruction::LShr: OpReg = X86::SHR64rCL; break;
+ case Instruction::AShr: OpReg = X86::SAR64rCL; break;
+ case Instruction::Shl: OpReg = X86::SHL64rCL; break;
+ default: return false;
+ }
+ } else {
+ return false;
+ }
+
+ MVT VT;
+ if (!isTypeLegal(I->getType(), VT))
+ return false;
+
+ unsigned Op0Reg = getRegForValue(I->getOperand(0));
+ if (Op0Reg == 0) return false;
+
+ unsigned Op1Reg = getRegForValue(I->getOperand(1));
+ if (Op1Reg == 0) return false;
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
+ CReg).addReg(Op1Reg);
+
+ // The shift instruction uses X86::CL. If we defined a super-register
+ // of X86::CL, emit a subreg KILL to precisely describe what we're doing here.
+ if (CReg != X86::CL)
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::KILL), X86::CL)
+ .addReg(CReg, RegState::Kill);
+
+ unsigned ResultReg = createResultReg(RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(OpReg), ResultReg)
+ .addReg(Op0Reg);
+ updateValueMap(I, ResultReg);
+ return true;
+}
+
+bool X86FastISel::X86SelectDivRem(const Instruction *I) {
+ const static unsigned NumTypes = 4; // i8, i16, i32, i64
+ const static unsigned NumOps = 4; // SDiv, SRem, UDiv, URem
+ const static bool S = true; // IsSigned
+ const static bool U = false; // !IsSigned
+ const static unsigned Copy = TargetOpcode::COPY;
+ // For the X86 DIV/IDIV instruction, in most cases the dividend
+ // (numerator) must be in a specific register pair highreg:lowreg,
+ // producing the quotient in lowreg and the remainder in highreg.
+ // For most data types, to set up the instruction, the dividend is
+ // copied into lowreg, and lowreg is sign-extended or zero-extended
+ // into highreg. The exception is i8, where the dividend is defined
+ // as a single register rather than a register pair, and we
+ // therefore directly sign-extend or zero-extend the dividend into
+ // lowreg, instead of copying, and ignore the highreg.
+ const static struct DivRemEntry {
+ // The following portion depends only on the data type.
+ const TargetRegisterClass *RC;
+ unsigned LowInReg; // low part of the register pair
+ unsigned HighInReg; // high part of the register pair
+ // The following portion depends on both the data type and the operation.
+ struct DivRemResult {
+ unsigned OpDivRem; // The specific DIV/IDIV opcode to use.
+ unsigned OpSignExtend; // Opcode for sign-extending lowreg into
+ // highreg, or copying a zero into highreg.
+ unsigned OpCopy; // Opcode for copying dividend into lowreg, or
+ // zero/sign-extending into lowreg for i8.
+ unsigned DivRemResultReg; // Register containing the desired result.
+ bool IsOpSigned; // Whether to use signed or unsigned form.
+ } ResultTable[NumOps];
+ } OpTable[NumTypes] = {
+ { &X86::GR8RegClass, X86::AX, 0, {
+ { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S }, // SDiv
+ { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S }, // SRem
+ { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U }, // UDiv
+ { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U }, // URem
+ }
+ }, // i8
+ { &X86::GR16RegClass, X86::AX, X86::DX, {
+ { X86::IDIV16r, X86::CWD, Copy, X86::AX, S }, // SDiv
+ { X86::IDIV16r, X86::CWD, Copy, X86::DX, S }, // SRem
+ { X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U }, // UDiv
+ { X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U }, // URem
+ }
+ }, // i16
+ { &X86::GR32RegClass, X86::EAX, X86::EDX, {
+ { X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S }, // SDiv
+ { X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S }, // SRem
+ { X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U }, // UDiv
+ { X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U }, // URem
+ }
+ }, // i32
+ { &X86::GR64RegClass, X86::RAX, X86::RDX, {
+ { X86::IDIV64r, X86::CQO, Copy, X86::RAX, S }, // SDiv
+ { X86::IDIV64r, X86::CQO, Copy, X86::RDX, S }, // SRem
+ { X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U }, // UDiv
+ { X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U }, // URem
+ }
+ }, // i64
+ };
+
+ MVT VT;
+ if (!isTypeLegal(I->getType(), VT))
+ return false;
+
+ unsigned TypeIndex, OpIndex;
+ switch (VT.SimpleTy) {
+ default: return false;
+ case MVT::i8: TypeIndex = 0; break;
+ case MVT::i16: TypeIndex = 1; break;
+ case MVT::i32: TypeIndex = 2; break;
+ case MVT::i64: TypeIndex = 3;
+ if (!Subtarget->is64Bit())
+ return false;
+ break;
+ }
+
+ switch (I->getOpcode()) {
+ default: llvm_unreachable("Unexpected div/rem opcode");
+ case Instruction::SDiv: OpIndex = 0; break;
+ case Instruction::SRem: OpIndex = 1; break;
+ case Instruction::UDiv: OpIndex = 2; break;
+ case Instruction::URem: OpIndex = 3; break;
+ }
+
+ const DivRemEntry &TypeEntry = OpTable[TypeIndex];
+ const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
+ unsigned Op0Reg = getRegForValue(I->getOperand(0));
+ if (Op0Reg == 0)
+ return false;
+ unsigned Op1Reg = getRegForValue(I->getOperand(1));
+ if (Op1Reg == 0)
+ return false;
+
+ // Move op0 into low-order input register.
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(OpEntry.OpCopy), TypeEntry.LowInReg).addReg(Op0Reg);
+ // Zero-extend or sign-extend into high-order input register.
+ if (OpEntry.OpSignExtend) {
+ if (OpEntry.IsOpSigned)
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(OpEntry.OpSignExtend));
+ else {
+ unsigned Zero32 = createResultReg(&X86::GR32RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(X86::MOV32r0), Zero32);
+
+ // Copy the zero into the appropriate sub/super/identical physical
+ // register. Unfortunately the operations needed are not uniform enough
+ // to fit neatly into the table above.
+ if (VT.SimpleTy == MVT::i16) {
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Copy), TypeEntry.HighInReg)
+ .addReg(Zero32, 0, X86::sub_16bit);
+ } else if (VT.SimpleTy == MVT::i32) {
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Copy), TypeEntry.HighInReg)
+ .addReg(Zero32);
+ } else if (VT.SimpleTy == MVT::i64) {
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
+ .addImm(0).addReg(Zero32).addImm(X86::sub_32bit);
+ }
+ }
+ }
+ // Generate the DIV/IDIV instruction.
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(OpEntry.OpDivRem)).addReg(Op1Reg);
+ // For i8 remainder, we can't reference AH directly, as we'll end
+ // up with bogus copies like %R9B = COPY %AH. Reference AX
+ // instead to prevent AH references in a REX instruction.
+ //
+ // The current assumption of the fast register allocator is that isel
+ // won't generate explicit references to the GPR8_NOREX registers. If
+ // the allocator and/or the backend get enhanced to be more robust in
+ // that regard, this can be, and should be, removed.
+ unsigned ResultReg = 0;
+ if ((I->getOpcode() == Instruction::SRem ||
+ I->getOpcode() == Instruction::URem) &&
+ OpEntry.DivRemResultReg == X86::AH && Subtarget->is64Bit()) {
+ unsigned SourceSuperReg = createResultReg(&X86::GR16RegClass);
+ unsigned ResultSuperReg = createResultReg(&X86::GR16RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Copy), SourceSuperReg).addReg(X86::AX);
+
+ // Shift AX right by 8 bits instead of using AH.
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::SHR16ri),
+ ResultSuperReg).addReg(SourceSuperReg).addImm(8);
+
+ // Now reference the 8-bit subreg of the result.
+ ResultReg = fastEmitInst_extractsubreg(MVT::i8, ResultSuperReg,
+ /*Kill=*/true, X86::sub_8bit);
+ }
+ // Copy the result out of the physreg if we haven't already.
+ if (!ResultReg) {
+ ResultReg = createResultReg(TypeEntry.RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Copy), ResultReg)
+ .addReg(OpEntry.DivRemResultReg);
+ }
+ updateValueMap(I, ResultReg);
+
+ return true;
+}
+
+/// \brief Emit a conditional move instruction (if the are supported) to lower
+/// the select.
+bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I) {
+ // Check if the subtarget supports these instructions.
+ if (!Subtarget->hasCMov())
+ return false;
+
+ // FIXME: Add support for i8.
+ if (RetVT < MVT::i16 || RetVT > MVT::i64)
+ return false;
+
+ const Value *Cond = I->getOperand(0);
+ const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
+ bool NeedTest = true;
+ X86::CondCode CC = X86::COND_NE;
+
+ // Optimize conditions coming from a compare if both instructions are in the
+ // same basic block (values defined in other basic blocks may not have
+ // initialized registers).
+ const auto *CI = dyn_cast<CmpInst>(Cond);
+ if (CI && (CI->getParent() == I->getParent())) {
+ CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
+
+ // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
+ static unsigned SETFOpcTable[2][3] = {
+ { X86::SETNPr, X86::SETEr , X86::TEST8rr },
+ { X86::SETPr, X86::SETNEr, X86::OR8rr }
+ };
+ unsigned *SETFOpc = nullptr;
+ switch (Predicate) {
+ default: break;
+ case CmpInst::FCMP_OEQ:
+ SETFOpc = &SETFOpcTable[0][0];
+ Predicate = CmpInst::ICMP_NE;
+ break;
+ case CmpInst::FCMP_UNE:
+ SETFOpc = &SETFOpcTable[1][0];
+ Predicate = CmpInst::ICMP_NE;
+ break;
+ }
+
+ bool NeedSwap;
+ std::tie(CC, NeedSwap) = getX86ConditionCode(Predicate);
+ assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
+
+ const Value *CmpLHS = CI->getOperand(0);
+ const Value *CmpRHS = CI->getOperand(1);
+ if (NeedSwap)
+ std::swap(CmpLHS, CmpRHS);
+
+ EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType());
+ // Emit a compare of the LHS and RHS, setting the flags.
+ if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))
+ return false;
+
+ if (SETFOpc) {
+ unsigned FlagReg1 = createResultReg(&X86::GR8RegClass);
+ unsigned FlagReg2 = createResultReg(&X86::GR8RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[0]),
+ FlagReg1);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[1]),
+ FlagReg2);
+ auto const &II = TII.get(SETFOpc[2]);
+ if (II.getNumDefs()) {
+ unsigned TmpReg = createResultReg(&X86::GR8RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, TmpReg)
+ .addReg(FlagReg2).addReg(FlagReg1);
+ } else {
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
+ .addReg(FlagReg2).addReg(FlagReg1);
+ }
+ }
+ NeedTest = false;
+ } else if (foldX86XALUIntrinsic(CC, I, Cond)) {
+ // Fake request the condition, otherwise the intrinsic might be completely
+ // optimized away.
+ unsigned TmpReg = getRegForValue(Cond);
+ if (TmpReg == 0)
+ return false;
+
+ NeedTest = false;
+ }
+
+ if (NeedTest) {
+ // Selects operate on i1, however, CondReg is 8 bits width and may contain
+ // garbage. Indeed, only the less significant bit is supposed to be
+ // accurate. If we read more than the lsb, we may see non-zero values
+ // whereas lsb is zero. Therefore, we have to truncate Op0Reg to i1 for
+ // the select. This is achieved by performing TEST against 1.
+ unsigned CondReg = getRegForValue(Cond);
+ if (CondReg == 0)
+ return false;
+ bool CondIsKill = hasTrivialKill(Cond);
+
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TEST8ri))
+ .addReg(CondReg, getKillRegState(CondIsKill)).addImm(1);
+ }
+
+ const Value *LHS = I->getOperand(1);
+ const Value *RHS = I->getOperand(2);
+
+ unsigned RHSReg = getRegForValue(RHS);
+ bool RHSIsKill = hasTrivialKill(RHS);
+
+ unsigned LHSReg = getRegForValue(LHS);
+ bool LHSIsKill = hasTrivialKill(LHS);
+
+ if (!LHSReg || !RHSReg)
+ return false;
+
+ unsigned Opc = X86::getCMovFromCond(CC, RC->getSize());
+ unsigned ResultReg = fastEmitInst_rr(Opc, RC, RHSReg, RHSIsKill,
+ LHSReg, LHSIsKill);
+ updateValueMap(I, ResultReg);
+ return true;
+}
+
+/// \brief Emit SSE or AVX instructions to lower the select.
+///
+/// Try to use SSE1/SSE2 instructions to simulate a select without branches.
+/// This lowers fp selects into a CMP/AND/ANDN/OR sequence when the necessary
+/// SSE instructions are available. If AVX is available, try to use a VBLENDV.
+bool X86FastISel::X86FastEmitSSESelect(MVT RetVT, const Instruction *I) {
+ // Optimize conditions coming from a compare if both instructions are in the
+ // same basic block (values defined in other basic blocks may not have
+ // initialized registers).
+ const auto *CI = dyn_cast<FCmpInst>(I->getOperand(0));
+ if (!CI || (CI->getParent() != I->getParent()))
+ return false;
+
+ if (I->getType() != CI->getOperand(0)->getType() ||
+ !((Subtarget->hasSSE1() && RetVT == MVT::f32) ||
+ (Subtarget->hasSSE2() && RetVT == MVT::f64)))
+ return false;
+
+ const Value *CmpLHS = CI->getOperand(0);
+ const Value *CmpRHS = CI->getOperand(1);
+ CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
+
+ // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
+ // We don't have to materialize a zero constant for this case and can just use
+ // %x again on the RHS.
+ if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
+ const auto *CmpRHSC = dyn_cast<ConstantFP>(CmpRHS);
+ if (CmpRHSC && CmpRHSC->isNullValue())
+ CmpRHS = CmpLHS;
+ }
+
+ unsigned CC;
+ bool NeedSwap;
+ std::tie(CC, NeedSwap) = getX86SSEConditionCode(Predicate);
+ if (CC > 7)
+ return false;
+
+ if (NeedSwap)
+ std::swap(CmpLHS, CmpRHS);
+
+ // Choose the SSE instruction sequence based on data type (float or double).
+ static unsigned OpcTable[2][4] = {
+ { X86::CMPSSrr, X86::FsANDPSrr, X86::FsANDNPSrr, X86::FsORPSrr },
+ { X86::CMPSDrr, X86::FsANDPDrr, X86::FsANDNPDrr, X86::FsORPDrr }
+ };
+
+ unsigned *Opc = nullptr;
+ switch (RetVT.SimpleTy) {
+ default: return false;
+ case MVT::f32: Opc = &OpcTable[0][0]; break;
+ case MVT::f64: Opc = &OpcTable[1][0]; break;
+ }
+
+ const Value *LHS = I->getOperand(1);
+ const Value *RHS = I->getOperand(2);
+
+ unsigned LHSReg = getRegForValue(LHS);
+ bool LHSIsKill = hasTrivialKill(LHS);
+
+ unsigned RHSReg = getRegForValue(RHS);
+ bool RHSIsKill = hasTrivialKill(RHS);
+
+ unsigned CmpLHSReg = getRegForValue(CmpLHS);
+ bool CmpLHSIsKill = hasTrivialKill(CmpLHS);
+
+ unsigned CmpRHSReg = getRegForValue(CmpRHS);
+ bool CmpRHSIsKill = hasTrivialKill(CmpRHS);
+
+ if (!LHSReg || !RHSReg || !CmpLHS || !CmpRHS)
+ return false;
+
+ const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
+ unsigned ResultReg;
+
+ if (Subtarget->hasAVX()) {
+ const TargetRegisterClass *FR32 = &X86::FR32RegClass;
+ const TargetRegisterClass *VR128 = &X86::VR128RegClass;
+
+ // If we have AVX, create 1 blendv instead of 3 logic instructions.
+ // Blendv was introduced with SSE 4.1, but the 2 register form implicitly
+ // uses XMM0 as the selection register. That may need just as many
+ // instructions as the AND/ANDN/OR sequence due to register moves, so
+ // don't bother.
+ unsigned CmpOpcode =
+ (RetVT.SimpleTy == MVT::f32) ? X86::VCMPSSrr : X86::VCMPSDrr;
+ unsigned BlendOpcode =
+ (RetVT.SimpleTy == MVT::f32) ? X86::VBLENDVPSrr : X86::VBLENDVPDrr;
+
+ unsigned CmpReg = fastEmitInst_rri(CmpOpcode, FR32, CmpLHSReg, CmpLHSIsKill,
+ CmpRHSReg, CmpRHSIsKill, CC);
+ unsigned VBlendReg = fastEmitInst_rrr(BlendOpcode, VR128, RHSReg, RHSIsKill,
+ LHSReg, LHSIsKill, CmpReg, true);
+ ResultReg = createResultReg(RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), ResultReg).addReg(VBlendReg);
+ } else {
+ unsigned CmpReg = fastEmitInst_rri(Opc[0], RC, CmpLHSReg, CmpLHSIsKill,
+ CmpRHSReg, CmpRHSIsKill, CC);
+ unsigned AndReg = fastEmitInst_rr(Opc[1], RC, CmpReg, /*IsKill=*/false,
+ LHSReg, LHSIsKill);
+ unsigned AndNReg = fastEmitInst_rr(Opc[2], RC, CmpReg, /*IsKill=*/true,
+ RHSReg, RHSIsKill);
+ ResultReg = fastEmitInst_rr(Opc[3], RC, AndNReg, /*IsKill=*/true,
+ AndReg, /*IsKill=*/true);
+ }
+ updateValueMap(I, ResultReg);
+ return true;
+}
+
+bool X86FastISel::X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I) {
+ // These are pseudo CMOV instructions and will be later expanded into control-
+ // flow.
+ unsigned Opc;
+ switch (RetVT.SimpleTy) {
+ default: return false;
+ case MVT::i8: Opc = X86::CMOV_GR8; break;
+ case MVT::i16: Opc = X86::CMOV_GR16; break;
+ case MVT::i32: Opc = X86::CMOV_GR32; break;
+ case MVT::f32: Opc = X86::CMOV_FR32; break;
+ case MVT::f64: Opc = X86::CMOV_FR64; break;
+ }
+
+ const Value *Cond = I->getOperand(0);
+ X86::CondCode CC = X86::COND_NE;
+
+ // Optimize conditions coming from a compare if both instructions are in the
+ // same basic block (values defined in other basic blocks may not have
+ // initialized registers).
+ const auto *CI = dyn_cast<CmpInst>(Cond);
+ if (CI && (CI->getParent() == I->getParent())) {
+ bool NeedSwap;
+ std::tie(CC, NeedSwap) = getX86ConditionCode(CI->getPredicate());
+ if (CC > X86::LAST_VALID_COND)
+ return false;
+
+ const Value *CmpLHS = CI->getOperand(0);
+ const Value *CmpRHS = CI->getOperand(1);
+
+ if (NeedSwap)
+ std::swap(CmpLHS, CmpRHS);
+
+ EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType());
+ if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))
+ return false;
+ } else {
+ unsigned CondReg = getRegForValue(Cond);
+ if (CondReg == 0)
+ return false;
+ bool CondIsKill = hasTrivialKill(Cond);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TEST8ri))
+ .addReg(CondReg, getKillRegState(CondIsKill)).addImm(1);
+ }
+
+ const Value *LHS = I->getOperand(1);
+ const Value *RHS = I->getOperand(2);
+
+ unsigned LHSReg = getRegForValue(LHS);
+ bool LHSIsKill = hasTrivialKill(LHS);
+
+ unsigned RHSReg = getRegForValue(RHS);
+ bool RHSIsKill = hasTrivialKill(RHS);
+
+ if (!LHSReg || !RHSReg)
+ return false;
+
+ const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
+
+ unsigned ResultReg =
+ fastEmitInst_rri(Opc, RC, RHSReg, RHSIsKill, LHSReg, LHSIsKill, CC);
+ updateValueMap(I, ResultReg);
+ return true;
+}
+
+bool X86FastISel::X86SelectSelect(const Instruction *I) {
+ MVT RetVT;
+ if (!isTypeLegal(I->getType(), RetVT))
+ return false;
+
+ // Check if we can fold the select.
+ if (const auto *CI = dyn_cast<CmpInst>(I->getOperand(0))) {
+ CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
+ const Value *Opnd = nullptr;
+ switch (Predicate) {
+ default: break;
+ case CmpInst::FCMP_FALSE: Opnd = I->getOperand(2); break;
+ case CmpInst::FCMP_TRUE: Opnd = I->getOperand(1); break;
+ }
+ // No need for a select anymore - this is an unconditional move.
+ if (Opnd) {
+ unsigned OpReg = getRegForValue(Opnd);
+ if (OpReg == 0)
+ return false;
+ bool OpIsKill = hasTrivialKill(Opnd);
+ const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
+ unsigned ResultReg = createResultReg(RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), ResultReg)
+ .addReg(OpReg, getKillRegState(OpIsKill));
+ updateValueMap(I, ResultReg);
+ return true;
+ }
+ }
+
+ // First try to use real conditional move instructions.
+ if (X86FastEmitCMoveSelect(RetVT, I))
+ return true;
+
+ // Try to use a sequence of SSE instructions to simulate a conditional move.
+ if (X86FastEmitSSESelect(RetVT, I))
+ return true;
+
+ // Fall-back to pseudo conditional move instructions, which will be later
+ // converted to control-flow.
+ if (X86FastEmitPseudoSelect(RetVT, I))
+ return true;
+
+ return false;
+}
+
+bool X86FastISel::X86SelectSIToFP(const Instruction *I) {
+ // The target-independent selection algorithm in FastISel already knows how
+ // to select a SINT_TO_FP if the target is SSE but not AVX.
+ // Early exit if the subtarget doesn't have AVX.
+ if (!Subtarget->hasAVX())
+ return false;
+
+ if (!I->getOperand(0)->getType()->isIntegerTy(32))
+ return false;
+
+ // Select integer to float/double conversion.
+ unsigned OpReg = getRegForValue(I->getOperand(0));
+ if (OpReg == 0)
+ return false;
+
+ const TargetRegisterClass *RC = nullptr;
+ unsigned Opcode;
+
+ if (I->getType()->isDoubleTy()) {
+ // sitofp int -> double
+ Opcode = X86::VCVTSI2SDrr;
+ RC = &X86::FR64RegClass;
+ } else if (I->getType()->isFloatTy()) {
+ // sitofp int -> float
+ Opcode = X86::VCVTSI2SSrr;
+ RC = &X86::FR32RegClass;
+ } else
+ return false;
+
+ unsigned ImplicitDefReg = createResultReg(RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
+ unsigned ResultReg =
+ fastEmitInst_rr(Opcode, RC, ImplicitDefReg, true, OpReg, false);
+ updateValueMap(I, ResultReg);
+ return true;
+}
+
+// Helper method used by X86SelectFPExt and X86SelectFPTrunc.
+bool X86FastISel::X86SelectFPExtOrFPTrunc(const Instruction *I,
+ unsigned TargetOpc,
+ const TargetRegisterClass *RC) {
+ assert((I->getOpcode() == Instruction::FPExt ||
+ I->getOpcode() == Instruction::FPTrunc) &&
+ "Instruction must be an FPExt or FPTrunc!");
+
+ unsigned OpReg = getRegForValue(I->getOperand(0));
+ if (OpReg == 0)
+ return false;
+
+ unsigned ResultReg = createResultReg(RC);
+ MachineInstrBuilder MIB;
+ MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpc),
+ ResultReg);
+ if (Subtarget->hasAVX())
+ MIB.addReg(OpReg);
+ MIB.addReg(OpReg);
+ updateValueMap(I, ResultReg);
+ return true;
+}
+
+bool X86FastISel::X86SelectFPExt(const Instruction *I) {
+ if (X86ScalarSSEf64 && I->getType()->isDoubleTy() &&
+ I->getOperand(0)->getType()->isFloatTy()) {
+ // fpext from float to double.
+ unsigned Opc = Subtarget->hasAVX() ? X86::VCVTSS2SDrr : X86::CVTSS2SDrr;
+ return X86SelectFPExtOrFPTrunc(I, Opc, &X86::FR64RegClass);
+ }
+
+ return false;
+}
+
+bool X86FastISel::X86SelectFPTrunc(const Instruction *I) {
+ if (X86ScalarSSEf64 && I->getType()->isFloatTy() &&
+ I->getOperand(0)->getType()->isDoubleTy()) {
+ // fptrunc from double to float.
+ unsigned Opc = Subtarget->hasAVX() ? X86::VCVTSD2SSrr : X86::CVTSD2SSrr;
+ return X86SelectFPExtOrFPTrunc(I, Opc, &X86::FR32RegClass);
+ }
+
+ return false;
+}
+
+bool X86FastISel::X86SelectTrunc(const Instruction *I) {
+ EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
+ EVT DstVT = TLI.getValueType(DL, I->getType());
+
+ // This code only handles truncation to byte.
+ if (DstVT != MVT::i8 && DstVT != MVT::i1)
+ return false;
+ if (!TLI.isTypeLegal(SrcVT))
+ return false;
+
+ unsigned InputReg = getRegForValue(I->getOperand(0));
+ if (!InputReg)
+ // Unhandled operand. Halt "fast" selection and bail.
+ return false;
+
+ if (SrcVT == MVT::i8) {
+ // Truncate from i8 to i1; no code needed.
+ updateValueMap(I, InputReg);
+ return true;
+ }
+
+ bool KillInputReg = false;
+ if (!Subtarget->is64Bit()) {
+ // If we're on x86-32; we can't extract an i8 from a general register.
+ // First issue a copy to GR16_ABCD or GR32_ABCD.
+ const TargetRegisterClass *CopyRC =
+ (SrcVT == MVT::i16) ? &X86::GR16_ABCDRegClass : &X86::GR32_ABCDRegClass;
+ unsigned CopyReg = createResultReg(CopyRC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), CopyReg).addReg(InputReg);
+ InputReg = CopyReg;
+ KillInputReg = true;
+ }
+
+ // Issue an extract_subreg.
+ unsigned ResultReg = fastEmitInst_extractsubreg(MVT::i8,
+ InputReg, KillInputReg,
+ X86::sub_8bit);
+ if (!ResultReg)
+ return false;
+
+ updateValueMap(I, ResultReg);
+ return true;
+}
+
+bool X86FastISel::IsMemcpySmall(uint64_t Len) {
+ return Len <= (Subtarget->is64Bit() ? 32 : 16);
+}
+
+bool X86FastISel::TryEmitSmallMemcpy(X86AddressMode DestAM,
+ X86AddressMode SrcAM, uint64_t Len) {
+
+ // Make sure we don't bloat code by inlining very large memcpy's.
+ if (!IsMemcpySmall(Len))
+ return false;
+
+ bool i64Legal = Subtarget->is64Bit();
+
+ // We don't care about alignment here since we just emit integer accesses.
+ while (Len) {
+ MVT VT;
+ if (Len >= 8 && i64Legal)
+ VT = MVT::i64;
+ else if (Len >= 4)
+ VT = MVT::i32;
+ else if (Len >= 2)
+ VT = MVT::i16;
+ else
+ VT = MVT::i8;
+
+ unsigned Reg;
+ bool RV = X86FastEmitLoad(VT, SrcAM, nullptr, Reg);
+ RV &= X86FastEmitStore(VT, Reg, /*Kill=*/true, DestAM);
+ assert(RV && "Failed to emit load or store??");
+
+ unsigned Size = VT.getSizeInBits()/8;
+ Len -= Size;
+ DestAM.Disp += Size;
+ SrcAM.Disp += Size;
+ }
+
+ return true;
+}
+
+bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
+ // FIXME: Handle more intrinsics.
+ switch (II->getIntrinsicID()) {
+ default: return false;
+ case Intrinsic::convert_from_fp16:
+ case Intrinsic::convert_to_fp16: {
+ if (Subtarget->useSoftFloat() || !Subtarget->hasF16C())
+ return false;
+
+ const Value *Op = II->getArgOperand(0);
+ unsigned InputReg = getRegForValue(Op);
+ if (InputReg == 0)
+ return false;
+
+ // F16C only allows converting from float to half and from half to float.
+ bool IsFloatToHalf = II->getIntrinsicID() == Intrinsic::convert_to_fp16;
+ if (IsFloatToHalf) {
+ if (!Op->getType()->isFloatTy())
+ return false;
+ } else {
+ if (!II->getType()->isFloatTy())
+ return false;
+ }
+
+ unsigned ResultReg = 0;
+ const TargetRegisterClass *RC = TLI.getRegClassFor(MVT::v8i16);
+ if (IsFloatToHalf) {
+ // 'InputReg' is implicitly promoted from register class FR32 to
+ // register class VR128 by method 'constrainOperandRegClass' which is
+ // directly called by 'fastEmitInst_ri'.
+ // Instruction VCVTPS2PHrr takes an extra immediate operand which is
+ // used to provide rounding control.
+ InputReg = fastEmitInst_ri(X86::VCVTPS2PHrr, RC, InputReg, false, 0);
+
+ // Move the lower 32-bits of ResultReg to another register of class GR32.
+ ResultReg = createResultReg(&X86::GR32RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(X86::VMOVPDI2DIrr), ResultReg)
+ .addReg(InputReg, RegState::Kill);
+
+ // The result value is in the lower 16-bits of ResultReg.
+ unsigned RegIdx = X86::sub_16bit;
+ ResultReg = fastEmitInst_extractsubreg(MVT::i16, ResultReg, true, RegIdx);
+ } else {
+ assert(Op->getType()->isIntegerTy(16) && "Expected a 16-bit integer!");
+ // Explicitly sign-extend the input to 32-bit.
+ InputReg = fastEmit_r(MVT::i16, MVT::i32, ISD::SIGN_EXTEND, InputReg,
+ /*Kill=*/false);
+
+ // The following SCALAR_TO_VECTOR will be expanded into a VMOVDI2PDIrr.
+ InputReg = fastEmit_r(MVT::i32, MVT::v4i32, ISD::SCALAR_TO_VECTOR,
+ InputReg, /*Kill=*/true);
+
+ InputReg = fastEmitInst_r(X86::VCVTPH2PSrr, RC, InputReg, /*Kill=*/true);
+
+ // The result value is in the lower 32-bits of ResultReg.
+ // Emit an explicit copy from register class VR128 to register class FR32.
+ ResultReg = createResultReg(&X86::FR32RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), ResultReg)
+ .addReg(InputReg, RegState::Kill);
+ }
+
+ updateValueMap(II, ResultReg);
+ return true;
+ }
+ case Intrinsic::frameaddress: {
+ MachineFunction *MF = FuncInfo.MF;
+ if (MF->getTarget().getMCAsmInfo()->usesWindowsCFI())
+ return false;
+
+ Type *RetTy = II->getCalledFunction()->getReturnType();
+
+ MVT VT;
+ if (!isTypeLegal(RetTy, VT))
+ return false;
+
+ unsigned Opc;
+ const TargetRegisterClass *RC = nullptr;
+
+ switch (VT.SimpleTy) {
+ default: llvm_unreachable("Invalid result type for frameaddress.");
+ case MVT::i32: Opc = X86::MOV32rm; RC = &X86::GR32RegClass; break;
+ case MVT::i64: Opc = X86::MOV64rm; RC = &X86::GR64RegClass; break;
+ }
+
+ // This needs to be set before we call getPtrSizedFrameRegister, otherwise
+ // we get the wrong frame register.
+ MachineFrameInfo *MFI = MF->getFrameInfo();
+ MFI->setFrameAddressIsTaken(true);
+
+ const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
+ unsigned FrameReg = RegInfo->getPtrSizedFrameRegister(*MF);
+ assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
+ (FrameReg == X86::EBP && VT == MVT::i32)) &&
+ "Invalid Frame Register!");
+
+ // Always make a copy of the frame register to to a vreg first, so that we
+ // never directly reference the frame register (the TwoAddressInstruction-
+ // Pass doesn't like that).
+ unsigned SrcReg = createResultReg(RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), SrcReg).addReg(FrameReg);
+
+ // Now recursively load from the frame address.
+ // movq (%rbp), %rax
+ // movq (%rax), %rax
+ // movq (%rax), %rax
+ // ...
+ unsigned DestReg;
+ unsigned Depth = cast<ConstantInt>(II->getOperand(0))->getZExtValue();
+ while (Depth--) {
+ DestReg = createResultReg(RC);
+ addDirectMem(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Opc), DestReg), SrcReg);
+ SrcReg = DestReg;
+ }
+
+ updateValueMap(II, SrcReg);
+ return true;
+ }
+ case Intrinsic::memcpy: {
+ const MemCpyInst *MCI = cast<MemCpyInst>(II);
+ // Don't handle volatile or variable length memcpys.
+ if (MCI->isVolatile())
+ return false;
+
+ if (isa<ConstantInt>(MCI->getLength())) {
+ // Small memcpy's are common enough that we want to do them
+ // without a call if possible.
+ uint64_t Len = cast<ConstantInt>(MCI->getLength())->getZExtValue();
+ if (IsMemcpySmall(Len)) {
+ X86AddressMode DestAM, SrcAM;
+ if (!X86SelectAddress(MCI->getRawDest(), DestAM) ||
+ !X86SelectAddress(MCI->getRawSource(), SrcAM))
+ return false;
+ TryEmitSmallMemcpy(DestAM, SrcAM, Len);
+ return true;
+ }
+ }
+
+ unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
+ if (!MCI->getLength()->getType()->isIntegerTy(SizeWidth))
+ return false;
+
+ if (MCI->getSourceAddressSpace() > 255 || MCI->getDestAddressSpace() > 255)
+ return false;
+
+ return lowerCallTo(II, "memcpy", II->getNumArgOperands() - 2);
+ }
+ case Intrinsic::memset: {
+ const MemSetInst *MSI = cast<MemSetInst>(II);
+
+ if (MSI->isVolatile())
+ return false;
+
+ unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
+ if (!MSI->getLength()->getType()->isIntegerTy(SizeWidth))
+ return false;
+
+ if (MSI->getDestAddressSpace() > 255)
+ return false;
+
+ return lowerCallTo(II, "memset", II->getNumArgOperands() - 2);
+ }
+ case Intrinsic::stackprotector: {
+ // Emit code to store the stack guard onto the stack.
+ EVT PtrTy = TLI.getPointerTy(DL);
+
+ const Value *Op1 = II->getArgOperand(0); // The guard's value.
+ const AllocaInst *Slot = cast<AllocaInst>(II->getArgOperand(1));
+
+ MFI.setStackProtectorIndex(FuncInfo.StaticAllocaMap[Slot]);
+
+ // Grab the frame index.
+ X86AddressMode AM;
+ if (!X86SelectAddress(Slot, AM)) return false;
+ if (!X86FastEmitStore(PtrTy, Op1, AM)) return false;
+ return true;
+ }
+ case Intrinsic::dbg_declare: {
+ const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
+ X86AddressMode AM;
+ assert(DI->getAddress() && "Null address should be checked earlier!");
+ if (!X86SelectAddress(DI->getAddress(), AM))
+ return false;
+ const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
+ // FIXME may need to add RegState::Debug to any registers produced,
+ // although ESP/EBP should be the only ones at the moment.
+ assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) &&
+ "Expected inlined-at fields to agree");
+ addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II), AM)
+ .addImm(0)
+ .addMetadata(DI->getVariable())
+ .addMetadata(DI->getExpression());
+ return true;
+ }
+ case Intrinsic::trap: {
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TRAP));
+ return true;
+ }
+ case Intrinsic::sqrt: {
+ if (!Subtarget->hasSSE1())
+ return false;
+
+ Type *RetTy = II->getCalledFunction()->getReturnType();
+
+ MVT VT;
+ if (!isTypeLegal(RetTy, VT))
+ return false;
+
+ // Unfortunately we can't use fastEmit_r, because the AVX version of FSQRT
+ // is not generated by FastISel yet.
+ // FIXME: Update this code once tablegen can handle it.
+ static const unsigned SqrtOpc[2][2] = {
+ {X86::SQRTSSr, X86::VSQRTSSr},
+ {X86::SQRTSDr, X86::VSQRTSDr}
+ };
+ bool HasAVX = Subtarget->hasAVX();
+ unsigned Opc;
+ const TargetRegisterClass *RC;
+ switch (VT.SimpleTy) {
+ default: return false;
+ case MVT::f32: Opc = SqrtOpc[0][HasAVX]; RC = &X86::FR32RegClass; break;
+ case MVT::f64: Opc = SqrtOpc[1][HasAVX]; RC = &X86::FR64RegClass; break;
+ }
+
+ const Value *SrcVal = II->getArgOperand(0);
+ unsigned SrcReg = getRegForValue(SrcVal);
+
+ if (SrcReg == 0)
+ return false;
+
+ unsigned ImplicitDefReg = 0;
+ if (HasAVX) {
+ ImplicitDefReg = createResultReg(RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
+ }
+
+ unsigned ResultReg = createResultReg(RC);
+ MachineInstrBuilder MIB;
+ MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc),
+ ResultReg);
+
+ if (ImplicitDefReg)
+ MIB.addReg(ImplicitDefReg);
+
+ MIB.addReg(SrcReg);
+
+ updateValueMap(II, ResultReg);
+ return true;
+ }
+ case Intrinsic::sadd_with_overflow:
+ case Intrinsic::uadd_with_overflow:
+ case Intrinsic::ssub_with_overflow:
+ case Intrinsic::usub_with_overflow:
+ case Intrinsic::smul_with_overflow:
+ case Intrinsic::umul_with_overflow: {
+ // This implements the basic lowering of the xalu with overflow intrinsics
+ // into add/sub/mul followed by either seto or setb.
+ const Function *Callee = II->getCalledFunction();
+ auto *Ty = cast<StructType>(Callee->getReturnType());
+ Type *RetTy = Ty->getTypeAtIndex(0U);
+ Type *CondTy = Ty->getTypeAtIndex(1);
+
+ MVT VT;
+ if (!isTypeLegal(RetTy, VT))
+ return false;
+
+ if (VT < MVT::i8 || VT > MVT::i64)
+ return false;
+
+ const Value *LHS = II->getArgOperand(0);
+ const Value *RHS = II->getArgOperand(1);
+
+ // Canonicalize immediate to the RHS.
+ if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) &&
+ isCommutativeIntrinsic(II))
+ std::swap(LHS, RHS);
+
+ bool UseIncDec = false;
+ if (isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isOne())
+ UseIncDec = true;
+
+ unsigned BaseOpc, CondOpc;
+ switch (II->getIntrinsicID()) {
+ default: llvm_unreachable("Unexpected intrinsic!");
+ case Intrinsic::sadd_with_overflow:
+ BaseOpc = UseIncDec ? unsigned(X86ISD::INC) : unsigned(ISD::ADD);
+ CondOpc = X86::SETOr;
+ break;
+ case Intrinsic::uadd_with_overflow:
+ BaseOpc = ISD::ADD; CondOpc = X86::SETBr; break;
+ case Intrinsic::ssub_with_overflow:
+ BaseOpc = UseIncDec ? unsigned(X86ISD::DEC) : unsigned(ISD::SUB);
+ CondOpc = X86::SETOr;
+ break;
+ case Intrinsic::usub_with_overflow:
+ BaseOpc = ISD::SUB; CondOpc = X86::SETBr; break;
+ case Intrinsic::smul_with_overflow:
+ BaseOpc = X86ISD::SMUL; CondOpc = X86::SETOr; break;
+ case Intrinsic::umul_with_overflow:
+ BaseOpc = X86ISD::UMUL; CondOpc = X86::SETOr; break;
+ }
+
+ unsigned LHSReg = getRegForValue(LHS);
+ if (LHSReg == 0)
+ return false;
+ bool LHSIsKill = hasTrivialKill(LHS);
+
+ unsigned ResultReg = 0;
+ // Check if we have an immediate version.
+ if (const auto *CI = dyn_cast<ConstantInt>(RHS)) {
+ static const unsigned Opc[2][4] = {
+ { X86::INC8r, X86::INC16r, X86::INC32r, X86::INC64r },
+ { X86::DEC8r, X86::DEC16r, X86::DEC32r, X86::DEC64r }
+ };
+
+ if (BaseOpc == X86ISD::INC || BaseOpc == X86ISD::DEC) {
+ ResultReg = createResultReg(TLI.getRegClassFor(VT));
+ bool IsDec = BaseOpc == X86ISD::DEC;
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Opc[IsDec][VT.SimpleTy-MVT::i8]), ResultReg)
+ .addReg(LHSReg, getKillRegState(LHSIsKill));
+ } else
+ ResultReg = fastEmit_ri(VT, VT, BaseOpc, LHSReg, LHSIsKill,
+ CI->getZExtValue());
+ }
+
+ unsigned RHSReg;
+ bool RHSIsKill;
+ if (!ResultReg) {
+ RHSReg = getRegForValue(RHS);
+ if (RHSReg == 0)
+ return false;
+ RHSIsKill = hasTrivialKill(RHS);
+ ResultReg = fastEmit_rr(VT, VT, BaseOpc, LHSReg, LHSIsKill, RHSReg,
+ RHSIsKill);
+ }
+
+ // FastISel doesn't have a pattern for all X86::MUL*r and X86::IMUL*r. Emit
+ // it manually.
+ if (BaseOpc == X86ISD::UMUL && !ResultReg) {
+ static const unsigned MULOpc[] =
+ { X86::MUL8r, X86::MUL16r, X86::MUL32r, X86::MUL64r };
+ static const unsigned Reg[] = { X86::AL, X86::AX, X86::EAX, X86::RAX };
+ // First copy the first operand into RAX, which is an implicit input to
+ // the X86::MUL*r instruction.
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), Reg[VT.SimpleTy-MVT::i8])
+ .addReg(LHSReg, getKillRegState(LHSIsKill));
+ ResultReg = fastEmitInst_r(MULOpc[VT.SimpleTy-MVT::i8],
+ TLI.getRegClassFor(VT), RHSReg, RHSIsKill);
+ } else if (BaseOpc == X86ISD::SMUL && !ResultReg) {
+ static const unsigned MULOpc[] =
+ { X86::IMUL8r, X86::IMUL16rr, X86::IMUL32rr, X86::IMUL64rr };
+ if (VT == MVT::i8) {
+ // Copy the first operand into AL, which is an implicit input to the
+ // X86::IMUL8r instruction.
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), X86::AL)
+ .addReg(LHSReg, getKillRegState(LHSIsKill));
+ ResultReg = fastEmitInst_r(MULOpc[0], TLI.getRegClassFor(VT), RHSReg,
+ RHSIsKill);
+ } else
+ ResultReg = fastEmitInst_rr(MULOpc[VT.SimpleTy-MVT::i8],
+ TLI.getRegClassFor(VT), LHSReg, LHSIsKill,
+ RHSReg, RHSIsKill);
+ }
+
+ if (!ResultReg)
+ return false;
+
+ unsigned ResultReg2 = FuncInfo.CreateRegs(CondTy);
+ assert((ResultReg+1) == ResultReg2 && "Nonconsecutive result registers.");
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CondOpc),
+ ResultReg2);
+
+ updateValueMap(II, ResultReg, 2);
+ return true;
+ }
+ case Intrinsic::x86_sse_cvttss2si:
+ case Intrinsic::x86_sse_cvttss2si64:
+ case Intrinsic::x86_sse2_cvttsd2si:
+ case Intrinsic::x86_sse2_cvttsd2si64: {
+ bool IsInputDouble;
+ switch (II->getIntrinsicID()) {
+ default: llvm_unreachable("Unexpected intrinsic.");
+ case Intrinsic::x86_sse_cvttss2si:
+ case Intrinsic::x86_sse_cvttss2si64:
+ if (!Subtarget->hasSSE1())
+ return false;
+ IsInputDouble = false;
+ break;
+ case Intrinsic::x86_sse2_cvttsd2si:
+ case Intrinsic::x86_sse2_cvttsd2si64:
+ if (!Subtarget->hasSSE2())
+ return false;
+ IsInputDouble = true;
+ break;
+ }
+
+ Type *RetTy = II->getCalledFunction()->getReturnType();
+ MVT VT;
+ if (!isTypeLegal(RetTy, VT))
+ return false;
+
+ static const unsigned CvtOpc[2][2][2] = {
+ { { X86::CVTTSS2SIrr, X86::VCVTTSS2SIrr },
+ { X86::CVTTSS2SI64rr, X86::VCVTTSS2SI64rr } },
+ { { X86::CVTTSD2SIrr, X86::VCVTTSD2SIrr },
+ { X86::CVTTSD2SI64rr, X86::VCVTTSD2SI64rr } }
+ };
+ bool HasAVX = Subtarget->hasAVX();
+ unsigned Opc;
+ switch (VT.SimpleTy) {
+ default: llvm_unreachable("Unexpected result type.");
+ case MVT::i32: Opc = CvtOpc[IsInputDouble][0][HasAVX]; break;
+ case MVT::i64: Opc = CvtOpc[IsInputDouble][1][HasAVX]; break;
+ }
+
+ // Check if we can fold insertelement instructions into the convert.
+ const Value *Op = II->getArgOperand(0);
+ while (auto *IE = dyn_cast<InsertElementInst>(Op)) {
+ const Value *Index = IE->getOperand(2);
+ if (!isa<ConstantInt>(Index))
+ break;
+ unsigned Idx = cast<ConstantInt>(Index)->getZExtValue();
+
+ if (Idx == 0) {
+ Op = IE->getOperand(1);
+ break;
+ }
+ Op = IE->getOperand(0);
+ }
+
+ unsigned Reg = getRegForValue(Op);
+ if (Reg == 0)
+ return false;
+
+ unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
+ .addReg(Reg);
+
+ updateValueMap(II, ResultReg);
+ return true;
+ }
+ }
+}
+
+bool X86FastISel::fastLowerArguments() {
+ if (!FuncInfo.CanLowerReturn)
+ return false;
+
+ const Function *F = FuncInfo.Fn;
+ if (F->isVarArg())
+ return false;
+
+ CallingConv::ID CC = F->getCallingConv();
+ if (CC != CallingConv::C)
+ return false;
+
+ if (Subtarget->isCallingConvWin64(CC))
+ return false;
+
+ if (!Subtarget->is64Bit())
+ return false;
+
+ // Only handle simple cases. i.e. Up to 6 i32/i64 scalar arguments.
+ unsigned GPRCnt = 0;
+ unsigned FPRCnt = 0;
+ unsigned Idx = 0;
+ for (auto const &Arg : F->args()) {
+ // The first argument is at index 1.
+ ++Idx;
+ if (F->getAttributes().hasAttribute(Idx, Attribute::ByVal) ||
+ F->getAttributes().hasAttribute(Idx, Attribute::InReg) ||
+ F->getAttributes().hasAttribute(Idx, Attribute::StructRet) ||
+ F->getAttributes().hasAttribute(Idx, Attribute::Nest))
+ return false;
+
+ Type *ArgTy = Arg.getType();
+ if (ArgTy->isStructTy() || ArgTy->isArrayTy() || ArgTy->isVectorTy())
+ return false;
+
+ EVT ArgVT = TLI.getValueType(DL, ArgTy);
+ if (!ArgVT.isSimple()) return false;
+ switch (ArgVT.getSimpleVT().SimpleTy) {
+ default: return false;
+ case MVT::i32:
+ case MVT::i64:
+ ++GPRCnt;
+ break;
+ case MVT::f32:
+ case MVT::f64:
+ if (!Subtarget->hasSSE1())
+ return false;
+ ++FPRCnt;
+ break;
+ }
+
+ if (GPRCnt > 6)
+ return false;
+
+ if (FPRCnt > 8)
+ return false;
+ }
+
+ static const MCPhysReg GPR32ArgRegs[] = {
+ X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D
+ };
+ static const MCPhysReg GPR64ArgRegs[] = {
+ X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8 , X86::R9
+ };
+ static const MCPhysReg XMMArgRegs[] = {
+ X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
+ X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
+ };
+
+ unsigned GPRIdx = 0;
+ unsigned FPRIdx = 0;
+ for (auto const &Arg : F->args()) {
+ MVT VT = TLI.getSimpleValueType(DL, Arg.getType());
+ const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
+ unsigned SrcReg;
+ switch (VT.SimpleTy) {
+ default: llvm_unreachable("Unexpected value type.");
+ case MVT::i32: SrcReg = GPR32ArgRegs[GPRIdx++]; break;
+ case MVT::i64: SrcReg = GPR64ArgRegs[GPRIdx++]; break;
+ case MVT::f32: // fall-through
+ case MVT::f64: SrcReg = XMMArgRegs[FPRIdx++]; break;
+ }
+ unsigned DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);
+ // FIXME: Unfortunately it's necessary to emit a copy from the livein copy.
+ // Without this, EmitLiveInCopies may eliminate the livein if its only
+ // use is a bitcast (which isn't turned into an instruction).
+ unsigned ResultReg = createResultReg(RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), ResultReg)
+ .addReg(DstReg, getKillRegState(true));
+ updateValueMap(&Arg, ResultReg);
+ }
+ return true;
+}
+
+static unsigned computeBytesPoppedByCallee(const X86Subtarget *Subtarget,
+ CallingConv::ID CC,
+ ImmutableCallSite *CS) {
+ if (Subtarget->is64Bit())
+ return 0;
+ if (Subtarget->getTargetTriple().isOSMSVCRT())
+ return 0;
+ if (CC == CallingConv::Fast || CC == CallingConv::GHC ||
+ CC == CallingConv::HiPE)
+ return 0;
+ if (CS && !CS->paramHasAttr(1, Attribute::StructRet))
+ return 0;
+ if (CS && CS->paramHasAttr(1, Attribute::InReg))
+ return 0;
+ return 4;
+}
+
+bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
+ auto &OutVals = CLI.OutVals;
+ auto &OutFlags = CLI.OutFlags;
+ auto &OutRegs = CLI.OutRegs;
+ auto &Ins = CLI.Ins;
+ auto &InRegs = CLI.InRegs;
+ CallingConv::ID CC = CLI.CallConv;
+ bool &IsTailCall = CLI.IsTailCall;
+ bool IsVarArg = CLI.IsVarArg;
+ const Value *Callee = CLI.Callee;
+ MCSymbol *Symbol = CLI.Symbol;
+
+ bool Is64Bit = Subtarget->is64Bit();
+ bool IsWin64 = Subtarget->isCallingConvWin64(CC);
+
+ // Handle only C, fastcc, and webkit_js calling conventions for now.
+ switch (CC) {
+ default: return false;
+ case CallingConv::C:
+ case CallingConv::Fast:
+ case CallingConv::WebKit_JS:
+ case CallingConv::X86_FastCall:
+ case CallingConv::X86_64_Win64:
+ case CallingConv::X86_64_SysV:
+ break;
+ }
+
+ // Allow SelectionDAG isel to handle tail calls.
+ if (IsTailCall)
+ return false;
+
+ // fastcc with -tailcallopt is intended to provide a guaranteed
+ // tail call optimization. Fastisel doesn't know how to do that.
+ if (CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt)
+ return false;
+
+ // Don't know how to handle Win64 varargs yet. Nothing special needed for
+ // x86-32. Special handling for x86-64 is implemented.
+ if (IsVarArg && IsWin64)
+ return false;
+
+ // Don't know about inalloca yet.
+ if (CLI.CS && CLI.CS->hasInAllocaArgument())
+ return false;
+
+ // Fast-isel doesn't know about callee-pop yet.
+ if (X86::isCalleePop(CC, Subtarget->is64Bit(), IsVarArg,
+ TM.Options.GuaranteedTailCallOpt))
+ return false;
+
+ SmallVector<MVT, 16> OutVTs;
+ SmallVector<unsigned, 16> ArgRegs;
+
+ // If this is a constant i1/i8/i16 argument, promote to i32 to avoid an extra
+ // instruction. This is safe because it is common to all FastISel supported
+ // calling conventions on x86.
+ for (int i = 0, e = OutVals.size(); i != e; ++i) {
+ Value *&Val = OutVals[i];
+ ISD::ArgFlagsTy Flags = OutFlags[i];
+ if (auto *CI = dyn_cast<ConstantInt>(Val)) {
+ if (CI->getBitWidth() < 32) {
+ if (Flags.isSExt())
+ Val = ConstantExpr::getSExt(CI, Type::getInt32Ty(CI->getContext()));
+ else
+ Val = ConstantExpr::getZExt(CI, Type::getInt32Ty(CI->getContext()));
+ }
+ }
+
+ // Passing bools around ends up doing a trunc to i1 and passing it.
+ // Codegen this as an argument + "and 1".
+ MVT VT;
+ auto *TI = dyn_cast<TruncInst>(Val);
+ unsigned ResultReg;
+ if (TI && TI->getType()->isIntegerTy(1) && CLI.CS &&
+ (TI->getParent() == CLI.CS->getInstruction()->getParent()) &&
+ TI->hasOneUse()) {
+ Value *PrevVal = TI->getOperand(0);
+ ResultReg = getRegForValue(PrevVal);
+
+ if (!ResultReg)
+ return false;
+
+ if (!isTypeLegal(PrevVal->getType(), VT))
+ return false;
+
+ ResultReg =
+ fastEmit_ri(VT, VT, ISD::AND, ResultReg, hasTrivialKill(PrevVal), 1);
+ } else {
+ if (!isTypeLegal(Val->getType(), VT))
+ return false;
+ ResultReg = getRegForValue(Val);
+ }
+
+ if (!ResultReg)
+ return false;
+
+ ArgRegs.push_back(ResultReg);
+ OutVTs.push_back(VT);
+ }
+
+ // Analyze operands of the call, assigning locations to each operand.
+ SmallVector<CCValAssign, 16> ArgLocs;
+ CCState CCInfo(CC, IsVarArg, *FuncInfo.MF, ArgLocs, CLI.RetTy->getContext());
+
+ // Allocate shadow area for Win64
+ if (IsWin64)
+ CCInfo.AllocateStack(32, 8);
+
+ CCInfo.AnalyzeCallOperands(OutVTs, OutFlags, CC_X86);
+
+ // Get a count of how many bytes are to be pushed on the stack.
+ unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
+
+ // Issue CALLSEQ_START
+ unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown))
+ .addImm(NumBytes).addImm(0);
+
+ // Walk the register/memloc assignments, inserting copies/loads.
+ const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
+ for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
+ CCValAssign const &VA = ArgLocs[i];
+ const Value *ArgVal = OutVals[VA.getValNo()];
+ MVT ArgVT = OutVTs[VA.getValNo()];
+
+ if (ArgVT == MVT::x86mmx)
+ return false;
+
+ unsigned ArgReg = ArgRegs[VA.getValNo()];
+
+ // Promote the value if needed.
+ switch (VA.getLocInfo()) {
+ case CCValAssign::Full: break;
+ case CCValAssign::SExt: {
+ assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
+ "Unexpected extend");
+ bool Emitted = X86FastEmitExtend(ISD::SIGN_EXTEND, VA.getLocVT(), ArgReg,
+ ArgVT, ArgReg);
+ assert(Emitted && "Failed to emit a sext!"); (void)Emitted;
+ ArgVT = VA.getLocVT();
+ break;
+ }
+ case CCValAssign::ZExt: {
+ assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
+ "Unexpected extend");
+ bool Emitted = X86FastEmitExtend(ISD::ZERO_EXTEND, VA.getLocVT(), ArgReg,
+ ArgVT, ArgReg);
+ assert(Emitted && "Failed to emit a zext!"); (void)Emitted;
+ ArgVT = VA.getLocVT();
+ break;
+ }
+ case CCValAssign::AExt: {
+ assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
+ "Unexpected extend");
+ bool Emitted = X86FastEmitExtend(ISD::ANY_EXTEND, VA.getLocVT(), ArgReg,
+ ArgVT, ArgReg);
+ if (!Emitted)
+ Emitted = X86FastEmitExtend(ISD::ZERO_EXTEND, VA.getLocVT(), ArgReg,
+ ArgVT, ArgReg);
+ if (!Emitted)
+ Emitted = X86FastEmitExtend(ISD::SIGN_EXTEND, VA.getLocVT(), ArgReg,
+ ArgVT, ArgReg);
+
+ assert(Emitted && "Failed to emit a aext!"); (void)Emitted;
+ ArgVT = VA.getLocVT();
+ break;
+ }
+ case CCValAssign::BCvt: {
+ ArgReg = fastEmit_r(ArgVT, VA.getLocVT(), ISD::BITCAST, ArgReg,
+ /*TODO: Kill=*/false);
+ assert(ArgReg && "Failed to emit a bitcast!");
+ ArgVT = VA.getLocVT();
+ break;
+ }
+ case CCValAssign::VExt:
+ // VExt has not been implemented, so this should be impossible to reach
+ // for now. However, fallback to Selection DAG isel once implemented.
+ return false;
+ case CCValAssign::AExtUpper:
+ case CCValAssign::SExtUpper:
+ case CCValAssign::ZExtUpper:
+ case CCValAssign::FPExt:
+ llvm_unreachable("Unexpected loc info!");
+ case CCValAssign::Indirect:
+ // FIXME: Indirect doesn't need extending, but fast-isel doesn't fully
+ // support this.
+ return false;
+ }
+
+ if (VA.isRegLoc()) {
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), VA.getLocReg()).addReg(ArgReg);
+ OutRegs.push_back(VA.getLocReg());
+ } else {
+ assert(VA.isMemLoc());
+
+ // Don't emit stores for undef values.
+ if (isa<UndefValue>(ArgVal))
+ continue;
+
+ unsigned LocMemOffset = VA.getLocMemOffset();
+ X86AddressMode AM;
+ AM.Base.Reg = RegInfo->getStackRegister();
+ AM.Disp = LocMemOffset;
+ ISD::ArgFlagsTy Flags = OutFlags[VA.getValNo()];
+ unsigned Alignment = DL.getABITypeAlignment(ArgVal->getType());
+ MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
+ MachinePointerInfo::getStack(*FuncInfo.MF, LocMemOffset),
+ MachineMemOperand::MOStore, ArgVT.getStoreSize(), Alignment);
+ if (Flags.isByVal()) {
+ X86AddressMode SrcAM;
+ SrcAM.Base.Reg = ArgReg;
+ if (!TryEmitSmallMemcpy(AM, SrcAM, Flags.getByValSize()))
+ return false;
+ } else if (isa<ConstantInt>(ArgVal) || isa<ConstantPointerNull>(ArgVal)) {
+ // If this is a really simple value, emit this with the Value* version
+ // of X86FastEmitStore. If it isn't simple, we don't want to do this,
+ // as it can cause us to reevaluate the argument.
+ if (!X86FastEmitStore(ArgVT, ArgVal, AM, MMO))
+ return false;
+ } else {
+ bool ValIsKill = hasTrivialKill(ArgVal);
+ if (!X86FastEmitStore(ArgVT, ArgReg, ValIsKill, AM, MMO))
+ return false;
+ }
+ }
+ }
+
+ // ELF / PIC requires GOT in the EBX register before function calls via PLT
+ // GOT pointer.
+ if (Subtarget->isPICStyleGOT()) {
+ unsigned Base = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), X86::EBX).addReg(Base);
+ }
+
+ if (Is64Bit && IsVarArg && !IsWin64) {
+ // From AMD64 ABI document:
+ // For calls that may call functions that use varargs or stdargs
+ // (prototype-less calls or calls to functions containing ellipsis (...) in
+ // the declaration) %al is used as hidden argument to specify the number
+ // of SSE registers used. The contents of %al do not need to match exactly
+ // the number of registers, but must be an ubound on the number of SSE
+ // registers used and is in the range 0 - 8 inclusive.
+
+ // Count the number of XMM registers allocated.
+ static const MCPhysReg XMMArgRegs[] = {
+ X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
+ X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
+ };
+ unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs);
+ assert((Subtarget->hasSSE1() || !NumXMMRegs)
+ && "SSE registers cannot be used when SSE is disabled");
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV8ri),
+ X86::AL).addImm(NumXMMRegs);
+ }
+
+ // Materialize callee address in a register. FIXME: GV address can be
+ // handled with a CALLpcrel32 instead.
+ X86AddressMode CalleeAM;
+ if (!X86SelectCallAddress(Callee, CalleeAM))
+ return false;
+
+ unsigned CalleeOp = 0;
+ const GlobalValue *GV = nullptr;
+ if (CalleeAM.GV != nullptr) {
+ GV = CalleeAM.GV;
+ } else if (CalleeAM.Base.Reg != 0) {
+ CalleeOp = CalleeAM.Base.Reg;
+ } else
+ return false;
+
+ // Issue the call.
+ MachineInstrBuilder MIB;
+ if (CalleeOp) {
+ // Register-indirect call.
+ unsigned CallOpc = Is64Bit ? X86::CALL64r : X86::CALL32r;
+ MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CallOpc))
+ .addReg(CalleeOp);
+ } else {
+ // Direct call.
+ assert(GV && "Not a direct call");
+ unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
+
+ // See if we need any target-specific flags on the GV operand.
+ unsigned char OpFlags = 0;
+
+ // On ELF targets, in both X86-64 and X86-32 mode, direct calls to
+ // external symbols most go through the PLT in PIC mode. If the symbol
+ // has hidden or protected visibility, or if it is static or local, then
+ // we don't need to use the PLT - we can directly call it.
+ if (Subtarget->isTargetELF() &&
+ TM.getRelocationModel() == Reloc::PIC_ &&
+ GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) {
+ OpFlags = X86II::MO_PLT;
+ } else if (Subtarget->isPICStyleStubAny() &&
+ !GV->isStrongDefinitionForLinker() &&
+ (!Subtarget->getTargetTriple().isMacOSX() ||
+ Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) {
+ // PC-relative references to external symbols should go through $stub,
+ // unless we're building with the leopard linker or later, which
+ // automatically synthesizes these stubs.
+ OpFlags = X86II::MO_DARWIN_STUB;
+ }
+
+ MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CallOpc));
+ if (Symbol)
+ MIB.addSym(Symbol, OpFlags);
+ else
+ MIB.addGlobalAddress(GV, 0, OpFlags);
+ }
+
+ // Add a register mask operand representing the call-preserved registers.
+ // Proper defs for return values will be added by setPhysRegsDeadExcept().
+ MIB.addRegMask(TRI.getCallPreservedMask(*FuncInfo.MF, CC));
+
+ // Add an implicit use GOT pointer in EBX.
+ if (Subtarget->isPICStyleGOT())
+ MIB.addReg(X86::EBX, RegState::Implicit);
+
+ if (Is64Bit && IsVarArg && !IsWin64)
+ MIB.addReg(X86::AL, RegState::Implicit);
+
+ // Add implicit physical register uses to the call.
+ for (auto Reg : OutRegs)
+ MIB.addReg(Reg, RegState::Implicit);
+
+ // Issue CALLSEQ_END
+ unsigned NumBytesForCalleeToPop =
+ computeBytesPoppedByCallee(Subtarget, CC, CLI.CS);
+ unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
+ .addImm(NumBytes).addImm(NumBytesForCalleeToPop);
+
+ // Now handle call return values.
+ SmallVector<CCValAssign, 16> RVLocs;
+ CCState CCRetInfo(CC, IsVarArg, *FuncInfo.MF, RVLocs,
+ CLI.RetTy->getContext());
+ CCRetInfo.AnalyzeCallResult(Ins, RetCC_X86);
+
+ // Copy all of the result registers out of their specified physreg.
+ unsigned ResultReg = FuncInfo.CreateRegs(CLI.RetTy);
+ for (unsigned i = 0; i != RVLocs.size(); ++i) {
+ CCValAssign &VA = RVLocs[i];
+ EVT CopyVT = VA.getValVT();
+ unsigned CopyReg = ResultReg + i;
+
+ // If this is x86-64, and we disabled SSE, we can't return FP values
+ if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&
+ ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) {
+ report_fatal_error("SSE register return with SSE disabled");
+ }
+
+ // If we prefer to use the value in xmm registers, copy it out as f80 and
+ // use a truncate to move it from fp stack reg to xmm reg.
+ if ((VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1) &&
+ isScalarFPTypeInSSEReg(VA.getValVT())) {
+ CopyVT = MVT::f80;
+ CopyReg = createResultReg(&X86::RFP80RegClass);
+ }
+
+ // Copy out the result.
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::COPY), CopyReg).addReg(VA.getLocReg());
+ InRegs.push_back(VA.getLocReg());
+
+ // Round the f80 to the right size, which also moves it to the appropriate
+ // xmm register. This is accomplished by storing the f80 value in memory
+ // and then loading it back.
+ if (CopyVT != VA.getValVT()) {
+ EVT ResVT = VA.getValVT();
+ unsigned Opc = ResVT == MVT::f32 ? X86::ST_Fp80m32 : X86::ST_Fp80m64;
+ unsigned MemSize = ResVT.getSizeInBits()/8;
+ int FI = MFI.CreateStackObject(MemSize, MemSize, false);
+ addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Opc)), FI)
+ .addReg(CopyReg);
+ Opc = ResVT == MVT::f32 ? X86::MOVSSrm : X86::MOVSDrm;
+ addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Opc), ResultReg + i), FI);
+ }
+ }
+
+ CLI.ResultReg = ResultReg;
+ CLI.NumResultRegs = RVLocs.size();
+ CLI.Call = MIB;
+
+ return true;
+}
+
+bool
+X86FastISel::fastSelectInstruction(const Instruction *I) {
+ switch (I->getOpcode()) {
+ default: break;
+ case Instruction::Load:
+ return X86SelectLoad(I);
+ case Instruction::Store:
+ return X86SelectStore(I);
+ case Instruction::Ret:
+ return X86SelectRet(I);
+ case Instruction::ICmp:
+ case Instruction::FCmp:
+ return X86SelectCmp(I);
+ case Instruction::ZExt:
+ return X86SelectZExt(I);
+ case Instruction::Br:
+ return X86SelectBranch(I);
+ case Instruction::LShr:
+ case Instruction::AShr:
+ case Instruction::Shl:
+ return X86SelectShift(I);
+ case Instruction::SDiv:
+ case Instruction::UDiv:
+ case Instruction::SRem:
+ case Instruction::URem:
+ return X86SelectDivRem(I);
+ case Instruction::Select:
+ return X86SelectSelect(I);
+ case Instruction::Trunc:
+ return X86SelectTrunc(I);
+ case Instruction::FPExt:
+ return X86SelectFPExt(I);
+ case Instruction::FPTrunc:
+ return X86SelectFPTrunc(I);
+ case Instruction::SIToFP:
+ return X86SelectSIToFP(I);
+ case Instruction::IntToPtr: // Deliberate fall-through.
+ case Instruction::PtrToInt: {
+ EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
+ EVT DstVT = TLI.getValueType(DL, I->getType());
+ if (DstVT.bitsGT(SrcVT))
+ return X86SelectZExt(I);
+ if (DstVT.bitsLT(SrcVT))
+ return X86SelectTrunc(I);
+ unsigned Reg = getRegForValue(I->getOperand(0));
+ if (Reg == 0) return false;
+ updateValueMap(I, Reg);
+ return true;
+ }
+ case Instruction::BitCast: {
+ // Select SSE2/AVX bitcasts between 128/256 bit vector types.
+ if (!Subtarget->hasSSE2())
+ return false;
+
+ EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
+ EVT DstVT = TLI.getValueType(DL, I->getType());
+
+ if (!SrcVT.isSimple() || !DstVT.isSimple())
+ return false;
+
+ if (!SrcVT.is128BitVector() &&
+ !(Subtarget->hasAVX() && SrcVT.is256BitVector()))
+ return false;
+
+ unsigned Reg = getRegForValue(I->getOperand(0));
+ if (Reg == 0)
+ return false;
+
+ // No instruction is needed for conversion. Reuse the register used by
+ // the fist operand.
+ updateValueMap(I, Reg);
+ return true;
+ }
+ }
+
+ return false;
+}
+
+unsigned X86FastISel::X86MaterializeInt(const ConstantInt *CI, MVT VT) {
+ if (VT > MVT::i64)
+ return 0;
+
+ uint64_t Imm = CI->getZExtValue();
+ if (Imm == 0) {
+ unsigned SrcReg = fastEmitInst_(X86::MOV32r0, &X86::GR32RegClass);
+ switch (VT.SimpleTy) {
+ default: llvm_unreachable("Unexpected value type");
+ case MVT::i1:
+ case MVT::i8:
+ return fastEmitInst_extractsubreg(MVT::i8, SrcReg, /*Kill=*/true,
+ X86::sub_8bit);
+ case MVT::i16:
+ return fastEmitInst_extractsubreg(MVT::i16, SrcReg, /*Kill=*/true,
+ X86::sub_16bit);
+ case MVT::i32:
+ return SrcReg;
+ case MVT::i64: {
+ unsigned ResultReg = createResultReg(&X86::GR64RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::SUBREG_TO_REG), ResultReg)
+ .addImm(0).addReg(SrcReg).addImm(X86::sub_32bit);
+ return ResultReg;
+ }
+ }
+ }
+
+ unsigned Opc = 0;
+ switch (VT.SimpleTy) {
+ default: llvm_unreachable("Unexpected value type");
+ case MVT::i1: VT = MVT::i8; // fall-through
+ case MVT::i8: Opc = X86::MOV8ri; break;
+ case MVT::i16: Opc = X86::MOV16ri; break;
+ case MVT::i32: Opc = X86::MOV32ri; break;
+ case MVT::i64: {
+ if (isUInt<32>(Imm))
+ Opc = X86::MOV32ri;
+ else if (isInt<32>(Imm))
+ Opc = X86::MOV64ri32;
+ else
+ Opc = X86::MOV64ri;
+ break;
+ }
+ }
+ if (VT == MVT::i64 && Opc == X86::MOV32ri) {
+ unsigned SrcReg = fastEmitInst_i(Opc, &X86::GR32RegClass, Imm);
+ unsigned ResultReg = createResultReg(&X86::GR64RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(TargetOpcode::SUBREG_TO_REG), ResultReg)
+ .addImm(0).addReg(SrcReg).addImm(X86::sub_32bit);
+ return ResultReg;
+ }
+ return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
+}
+
+unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
+ if (CFP->isNullValue())
+ return fastMaterializeFloatZero(CFP);
+
+ // Can't handle alternate code models yet.
+ CodeModel::Model CM = TM.getCodeModel();
+ if (CM != CodeModel::Small && CM != CodeModel::Large)
+ return 0;
+
+ // Get opcode and regclass of the output for the given load instruction.
+ unsigned Opc = 0;
+ const TargetRegisterClass *RC = nullptr;
+ switch (VT.SimpleTy) {
+ default: return 0;
+ case MVT::f32:
+ if (X86ScalarSSEf32) {
+ Opc = Subtarget->hasAVX() ? X86::VMOVSSrm : X86::MOVSSrm;
+ RC = &X86::FR32RegClass;
+ } else {
+ Opc = X86::LD_Fp32m;
+ RC = &X86::RFP32RegClass;
+ }
+ break;
+ case MVT::f64:
+ if (X86ScalarSSEf64) {
+ Opc = Subtarget->hasAVX() ? X86::VMOVSDrm : X86::MOVSDrm;
+ RC = &X86::FR64RegClass;
+ } else {
+ Opc = X86::LD_Fp64m;
+ RC = &X86::RFP64RegClass;
+ }
+ break;
+ case MVT::f80:
+ // No f80 support yet.
+ return 0;
+ }
+
+ // MachineConstantPool wants an explicit alignment.
+ unsigned Align = DL.getPrefTypeAlignment(CFP->getType());
+ if (Align == 0) {
+ // Alignment of vector types. FIXME!
+ Align = DL.getTypeAllocSize(CFP->getType());
+ }
+
+ // x86-32 PIC requires a PIC base register for constant pools.
+ unsigned PICBase = 0;
+ unsigned char OpFlag = 0;
+ if (Subtarget->isPICStyleStubPIC()) { // Not dynamic-no-pic
+ OpFlag = X86II::MO_PIC_BASE_OFFSET;
+ PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
+ } else if (Subtarget->isPICStyleGOT()) {
+ OpFlag = X86II::MO_GOTOFF;
+ PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
+ } else if (Subtarget->isPICStyleRIPRel() &&
+ TM.getCodeModel() == CodeModel::Small) {
+ PICBase = X86::RIP;
+ }
+
+ // Create the load from the constant pool.
+ unsigned CPI = MCP.getConstantPoolIndex(CFP, Align);
+ unsigned ResultReg = createResultReg(RC);
+
+ if (CM == CodeModel::Large) {
+ unsigned AddrReg = createResultReg(&X86::GR64RegClass);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV64ri),
+ AddrReg)
+ .addConstantPoolIndex(CPI, 0, OpFlag);
+ MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Opc), ResultReg);
+ addDirectMem(MIB, AddrReg);
+ MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
+ MachinePointerInfo::getConstantPool(*FuncInfo.MF),
+ MachineMemOperand::MOLoad, DL.getPointerSize(), Align);
+ MIB->addMemOperand(*FuncInfo.MF, MMO);
+ return ResultReg;
+ }
+
+ addConstantPoolReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Opc), ResultReg),
+ CPI, PICBase, OpFlag);
+ return ResultReg;
+}
+
+unsigned X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
+ // Can't handle alternate code models yet.
+ if (TM.getCodeModel() != CodeModel::Small)
+ return 0;
+
+ // Materialize addresses with LEA/MOV instructions.
+ X86AddressMode AM;
+ if (X86SelectAddress(GV, AM)) {
+ // If the expression is just a basereg, then we're done, otherwise we need
+ // to emit an LEA.
+ if (AM.BaseType == X86AddressMode::RegBase &&
+ AM.IndexReg == 0 && AM.Disp == 0 && AM.GV == nullptr)
+ return AM.Base.Reg;
+
+ unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
+ if (TM.getRelocationModel() == Reloc::Static &&
+ TLI.getPointerTy(DL) == MVT::i64) {
+ // The displacement code could be more than 32 bits away so we need to use
+ // an instruction with a 64 bit immediate
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV64ri),
+ ResultReg)
+ .addGlobalAddress(GV);
+ } else {
+ unsigned Opc =
+ TLI.getPointerTy(DL) == MVT::i32
+ ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
+ : X86::LEA64r;
+ addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Opc), ResultReg), AM);
+ }
+ return ResultReg;
+ }
+ return 0;
+}
+
+unsigned X86FastISel::fastMaterializeConstant(const Constant *C) {
+ EVT CEVT = TLI.getValueType(DL, C->getType(), true);
+
+ // Only handle simple types.
+ if (!CEVT.isSimple())
+ return 0;
+ MVT VT = CEVT.getSimpleVT();
+
+ if (const auto *CI = dyn_cast<ConstantInt>(C))
+ return X86MaterializeInt(CI, VT);
+ else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
+ return X86MaterializeFP(CFP, VT);
+ else if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
+ return X86MaterializeGV(GV, VT);
+
+ return 0;
+}
+
+unsigned X86FastISel::fastMaterializeAlloca(const AllocaInst *C) {
+ // Fail on dynamic allocas. At this point, getRegForValue has already
+ // checked its CSE maps, so if we're here trying to handle a dynamic
+ // alloca, we're not going to succeed. X86SelectAddress has a
+ // check for dynamic allocas, because it's called directly from
+ // various places, but targetMaterializeAlloca also needs a check
+ // in order to avoid recursion between getRegForValue,
+ // X86SelectAddrss, and targetMaterializeAlloca.
+ if (!FuncInfo.StaticAllocaMap.count(C))
+ return 0;
+ assert(C->isStaticAlloca() && "dynamic alloca in the static alloca map?");
+
+ X86AddressMode AM;
+ if (!X86SelectAddress(C, AM))
+ return 0;
+ unsigned Opc =
+ TLI.getPointerTy(DL) == MVT::i32
+ ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
+ : X86::LEA64r;
+ const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
+ unsigned ResultReg = createResultReg(RC);
+ addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+ TII.get(Opc), ResultReg), AM);
+ return ResultReg;
+}
+
+unsigned X86FastISel::fastMaterializeFloatZero(const ConstantFP *CF) {
+ MVT VT;
+ if (!isTypeLegal(CF->getType(), VT))
+ return 0;
+
+ // Get opcode and regclass for the given zero.
+ unsigned Opc = 0;
+ const TargetRegisterClass *RC = nullptr;
+ switch (VT.SimpleTy) {
+ default: return 0;
+ case MVT::f32:
+ if (X86ScalarSSEf32) {
+ Opc = X86::FsFLD0SS;
+ RC = &X86::FR32RegClass;
+ } else {
+ Opc = X86::LD_Fp032;
+ RC = &X86::RFP32RegClass;
+ }
+ break;
+ case MVT::f64:
+ if (X86ScalarSSEf64) {
+ Opc = X86::FsFLD0SD;
+ RC = &X86::FR64RegClass;
+ } else {
+ Opc = X86::LD_Fp064;
+ RC = &X86::RFP64RegClass;
+ }
+ break;
+ case MVT::f80:
+ // No f80 support yet.
+ return 0;
+ }
+
+ unsigned ResultReg = createResultReg(RC);
+ BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg);
+ return ResultReg;
+}
+
+
+bool X86FastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
+ const LoadInst *LI) {
+ const Value *Ptr = LI->getPointerOperand();
+ X86AddressMode AM;
+ if (!X86SelectAddress(Ptr, AM))
+ return false;
+
+ const X86InstrInfo &XII = (const X86InstrInfo &)TII;
+
+ unsigned Size = DL.getTypeAllocSize(LI->getType());
+ unsigned Alignment = LI->getAlignment();
+
+ if (Alignment == 0) // Ensure that codegen never sees alignment 0
+ Alignment = DL.getABITypeAlignment(LI->getType());
+
+ SmallVector<MachineOperand, 8> AddrOps;
+ AM.getFullAddress(AddrOps);
+
+ MachineInstr *Result = XII.foldMemoryOperandImpl(
+ *FuncInfo.MF, MI, OpNo, AddrOps, FuncInfo.InsertPt, Size, Alignment,
+ /*AllowCommute=*/true);
+ if (!Result)
+ return false;
+
+ // The index register could be in the wrong register class. Unfortunately,
+ // foldMemoryOperandImpl could have commuted the instruction so its not enough
+ // to just look at OpNo + the offset to the index reg. We actually need to
+ // scan the instruction to find the index reg and see if its the correct reg
+ // class.
+ unsigned OperandNo = 0;
+ for (MachineInstr::mop_iterator I = Result->operands_begin(),
+ E = Result->operands_end(); I != E; ++I, ++OperandNo) {
+ MachineOperand &MO = *I;
+ if (!MO.isReg() || MO.isDef() || MO.getReg() != AM.IndexReg)
+ continue;
+ // Found the index reg, now try to rewrite it.
+ unsigned IndexReg = constrainOperandRegClass(Result->getDesc(),
+ MO.getReg(), OperandNo);
+ if (IndexReg == MO.getReg())
+ continue;
+ MO.setReg(IndexReg);
+ }
+
+ Result->addMemOperand(*FuncInfo.MF, createMachineMemOperandFor(LI));
+ MI->eraseFromParent();
+ return true;
+}
+
+
+namespace llvm {
+ FastISel *X86::createFastISel(FunctionLoweringInfo &funcInfo,
+ const TargetLibraryInfo *libInfo) {
+ return new X86FastISel(funcInfo, libInfo);
+ }
+}
projects / oota-llvm.git / blobdiff