From: Philip Reames Date: Mon, 1 Dec 2014 22:52:56 +0000 (+0000) Subject: [Statepoints 2/4] Statepoint infrastructure for garbage collection: MI & x86-64 Backend X-Git-Url: http://plrg.eecs.uci.edu/git/?p=oota-llvm.git;a=commitdiff_plain;h=78cc6fcb01a7306a380aeb65f411c89d6aa5532d [Statepoints 2/4] Statepoint infrastructure for garbage collection: MI & x86-64 Backend This is the second patch in a small series. This patch contains the MachineInstruction and x86-64 backend pieces required to lower Statepoints. It does not include the code to actually generate the STATEPOINT machine instruction and as a result, the entire patch is currently dead code. I will be submitting the SelectionDAG parts within the next 24-48 hours. Since those pieces are by far the most complicated, I wanted to minimize the size of that patch. That patch will include the tests which exercise the functionality in this patch. The entire series can be seen as one combined whole in http://reviews.llvm.org/D5683. The STATEPOINT psuedo node is generated after all gc values are explicitly spilled to stack slots. The purpose of this node is to wrap an actual call instruction while recording the spill locations of the meta arguments used for garbage collection and other purposes. The STATEPOINT is modeled as modifing all of those locations to prevent backend optimizations from forwarding the value from before the STATEPOINT to after the STATEPOINT. (Doing so would break relocation semantics for collectors which wish to relocate roots.) The implementation of STATEPOINT is closely modeled on PATCHPOINT. Eventually, much of the code in this patch will be removed. The long term plan is to merge the functionality provided by statepoints and patchpoints. Merging their implementations in the backend is likely to be a good starting point. Reviewed by: atrick, ributzka git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@223085 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h index e343980d2ab..4e48afe1400 100644 --- a/include/llvm/CodeGen/StackMaps.h +++ b/include/llvm/CodeGen/StackMaps.h @@ -81,6 +81,52 @@ public: unsigned getNextScratchIdx(unsigned StartIdx = 0) const; }; +/// MI-level Statepoint operands +/// +/// Statepoint operands take the form: +/// , , [call arguments], +/// , , +/// , , [other args], +/// [gc values] +class StatepointOpers { +private: + enum { + NCallArgsPos = 0, + CallTargetPos = 1 + }; + +public: + explicit StatepointOpers(const MachineInstr *MI): + MI(MI) { } + + /// Get starting index of non call related arguments + /// (statepoint flags, vm state and gc state). + unsigned getVarIdx() const { + return MI->getOperand(NCallArgsPos).getImm() + 2; + } + + /// Returns the index of the operand containing the number of non-gc non-call + /// arguments. + unsigned getNumVMSArgsIdx() const { + return getVarIdx() + 3; + } + + /// Returns the number of non-gc non-call arguments attached to the + /// statepoint. Note that this is the number of arguments, not the number of + /// operands required to represent those arguments. + unsigned getNumVMSArgs() const { + return MI->getOperand(getNumVMSArgsIdx()).getImm(); + } + + /// Returns the target of the underlying call. + const MachineOperand &getCallTarget() const { + return MI->getOperand(CallTargetPos); + } + +private: + const MachineInstr *MI; +}; + class StackMaps { public: struct Location { @@ -132,6 +178,9 @@ public: /// \brief Generate a stackmap record for a patchpoint instruction. void recordPatchPoint(const MachineInstr &MI); + /// \brief Generate a stackmap record for a statepoint instruction. + void recordStatepoint(const MachineInstr &MI); + /// If there is any stack map data, create a stack map section and serialize /// the map info into it. This clears the stack map data structures /// afterwards. @@ -139,7 +188,6 @@ public: private: static const char *WSMP; - typedef SmallVector LocationVec; typedef SmallVector LiveOutVec; typedef MapVector ConstantPool; diff --git a/include/llvm/Target/Target.td b/include/llvm/Target/Target.td index d28ce692a3a..688c03fd7e2 100644 --- a/include/llvm/Target/Target.td +++ b/include/llvm/Target/Target.td @@ -845,6 +845,15 @@ def PATCHPOINT : Instruction { let mayLoad = 1; let usesCustomInserter = 1; } +def STATEPOINT : Instruction { + let OutOperandList = (outs); + let InOperandList = (ins variable_ops); + let usesCustomInserter = 1; + let mayLoad = 1; + let mayStore = 1; + let hasSideEffects = 1; + let isCall = 1; +} def LOAD_STACK_GUARD : Instruction { let OutOperandList = (outs ptr_rc:$dst); let InOperandList = (ins); diff --git a/include/llvm/Target/TargetFrameLowering.h b/include/llvm/Target/TargetFrameLowering.h index bfddd060179..82c6093d8fb 100644 --- a/include/llvm/Target/TargetFrameLowering.h +++ b/include/llvm/Target/TargetFrameLowering.h @@ -199,6 +199,16 @@ public: virtual int getFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg) const; + /// Same as above, except that the 'base register' will always be RSP, not + /// RBP on x86. This is used exclusively for lowering STATEPOINT nodes. + /// TODO: This should really be a parameterizable choice. + virtual int getFrameIndexReferenceFromSP(const MachineFunction &MF, int FI, + unsigned &FrameReg) const { + // default to calling normal version, we override this on x86 only + llvm_unreachable("unimplemented for non-x86"); + return 0; + } + /// processFunctionBeforeCalleeSavedScan - This method is called immediately /// before PrologEpilogInserter scans the physical registers used to determine /// what callee saved registers should be spilled. This method is optional. diff --git a/include/llvm/Target/TargetOpcodes.h b/include/llvm/Target/TargetOpcodes.h index 1fbd2ae09b1..7ab1bc3837f 100644 --- a/include/llvm/Target/TargetOpcodes.h +++ b/include/llvm/Target/TargetOpcodes.h @@ -110,7 +110,13 @@ enum { /// to prevent the stack guard value or address from being spilled to the /// stack should override TargetLowering::emitLoadStackGuardNode and /// additionally expand this pseudo after register allocation. - LOAD_STACK_GUARD = 19 + LOAD_STACK_GUARD = 19, + + /// Call instruction with associated vm state for deoptimization and list + /// of live pointers for relocation by the garbage collector. It is + /// intended to support garbage collection with fully precise relocating + /// collectors and deoptimizations in either the callee or caller. + STATEPOINT = 20 }; } // end namespace TargetOpcode } // end namespace llvm diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp index 6a6e15dabff..121a89e0da2 100644 --- a/lib/CodeGen/InlineSpiller.cpp +++ b/lib/CodeGen/InlineSpiller.cpp @@ -1088,7 +1088,8 @@ foldMemoryOperand(ArrayRef > Ops, bool WasCopy = MI->isCopy(); unsigned ImpReg = 0; - bool SpillSubRegs = (MI->getOpcode() == TargetOpcode::PATCHPOINT || + bool SpillSubRegs = (MI->getOpcode() == TargetOpcode::STATEPOINT || + MI->getOpcode() == TargetOpcode::PATCHPOINT || MI->getOpcode() == TargetOpcode::STACKMAP); // TargetInstrInfo::foldMemoryOperand only expects explicit, non-tied diff --git a/lib/CodeGen/LocalStackSlotAllocation.cpp b/lib/CodeGen/LocalStackSlotAllocation.cpp index 5c5712f37e1..e8bf687a626 100644 --- a/lib/CodeGen/LocalStackSlotAllocation.cpp +++ b/lib/CodeGen/LocalStackSlotAllocation.cpp @@ -291,6 +291,7 @@ bool LocalStackSlotPass::insertFrameReferenceRegisters(MachineFunction &Fn) { // Debug value, stackmap and patchpoint instructions can't be out of // range, so they don't need any updates. if (MI->isDebugValue() || + MI->getOpcode() == TargetOpcode::STATEPOINT || MI->getOpcode() == TargetOpcode::STACKMAP || MI->getOpcode() == TargetOpcode::PATCHPOINT) continue; diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp index 06530b91d67..a19bd96b3f5 100644 --- a/lib/CodeGen/PrologEpilogInserter.cpp +++ b/lib/CodeGen/PrologEpilogInserter.cpp @@ -797,6 +797,26 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, continue; } + // TODO: This code should be commoned with the code for + // PATCHPOINT. There's no good reason for the difference in + // implementation other than historical accident. The only + // remaining difference is the unconditional use of the stack + // pointer as the base register. + if (MI->getOpcode() == TargetOpcode::STATEPOINT) { + assert((!MI->isDebugValue() || i == 0) && + "Frame indicies can only appear as the first operand of a " + "DBG_VALUE machine instruction"); + unsigned Reg; + MachineOperand &Offset = MI->getOperand(i + 1); + const unsigned refOffset = + TFI->getFrameIndexReferenceFromSP(Fn, MI->getOperand(i).getIndex(), + Reg); + + Offset.setImm(Offset.getImm() + refOffset); + MI->getOperand(i).ChangeToRegister(Reg, false /*isDef*/); + continue; + } + // Some instructions (e.g. inline asm instructions) can have // multiple frame indices and/or cause eliminateFrameIndex // to insert more than one instruction. We need the register diff --git a/lib/CodeGen/StackMaps.cpp b/lib/CodeGen/StackMaps.cpp index d3791c3feb5..aab8d522144 100644 --- a/lib/CodeGen/StackMaps.cpp +++ b/lib/CodeGen/StackMaps.cpp @@ -286,6 +286,18 @@ void StackMaps::recordPatchPoint(const MachineInstr &MI) { } #endif } +void StackMaps::recordStatepoint(const MachineInstr &MI) { + assert(MI.getOpcode() == TargetOpcode::STATEPOINT && + "expected statepoint"); + + StatepointOpers opers(&MI); + // Record all the deopt and gc operands (they're contiguous and run from the + // initial index to the end of the operand list) + const unsigned StartIdx = opers.getVarIdx(); + recordStackMapOpers(MI, 0xABCDEF00, + MI.operands_begin() + StartIdx, MI.operands_end(), + false); +} /// Emit the stackmap header. /// diff --git a/lib/CodeGen/TargetLoweringBase.cpp b/lib/CodeGen/TargetLoweringBase.cpp index bd5cd4c8cc2..8805104689e 100644 --- a/lib/CodeGen/TargetLoweringBase.cpp +++ b/lib/CodeGen/TargetLoweringBase.cpp @@ -992,8 +992,14 @@ TargetLoweringBase::emitPatchPoint(MachineInstr *MI, // Add a new memory operand for this FI. const MachineFrameInfo &MFI = *MF.getFrameInfo(); assert(MFI.getObjectOffset(FI) != -1); + + unsigned Flags = MachineMemOperand::MOLoad; + if (MI->getOpcode() == TargetOpcode::STATEPOINT) { + Flags |= MachineMemOperand::MOStore; + Flags |= MachineMemOperand::MOVolatile; + } MachineMemOperand *MMO = MF.getMachineMemOperand( - MachinePointerInfo::getFixedStack(FI), MachineMemOperand::MOLoad, + MachinePointerInfo::getFixedStack(FI), Flags, TM.getSubtargetImpl()->getDataLayout()->getPointerSize(), MFI.getObjectAlignment(FI)); MIB->addMemOperand(MF, MMO); diff --git a/lib/Target/X86/X86FrameLowering.cpp b/lib/Target/X86/X86FrameLowering.cpp index b9920b1b84c..278e4f910d4 100644 --- a/lib/Target/X86/X86FrameLowering.cpp +++ b/lib/Target/X86/X86FrameLowering.cpp @@ -1135,6 +1135,79 @@ int X86FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI, return getFrameIndexOffset(MF, FI); } +// Simplified from getFrameIndexOffset keeping only StackPointer cases +int X86FrameLowering::getFrameIndexOffsetFromSP(const MachineFunction &MF, int FI) const { + const X86RegisterInfo *RegInfo = + static_cast(MF.getSubtarget().getRegisterInfo()); + const MachineFrameInfo *MFI = MF.getFrameInfo(); + const uint64_t StackSize = MFI->getStackSize(); //not including dynamic realign + + { +#ifndef NDEBUG + // Note: LLVM arranges the stack as: + // Args > Saved RetPC (<--FP) > CSRs > dynamic alignment (<--BP) + // > "Stack Slots" (<--SP) + // We can always address StackSlots from RSP. We can usually (unless + // needsStackRealignment) address CSRs from RSP, but sometimes need to + // address them from RBP. FixedObjects can be placed anywhere in the stack + // frame depending on their specific requirements (i.e. we can actually + // refer to arguments to the function which are stored in the *callers* + // frame). As a result, THE RESULT OF THIS CALL IS MEANINGLESS FOR CSRs + // AND FixedObjects IFF needsStackRealignment or hasVarSizedObject. + + assert(!RegInfo->hasBasePointer(MF) && "we don't handle this case"); + + // We don't handle tail calls, and shouldn't be seeing them + // either. + int TailCallReturnAddrDelta = + MF.getInfo()->getTCReturnAddrDelta(); + assert(!(TailCallReturnAddrDelta < 0) && "we don't handle this case!"); +#endif + } + + // This is how the math works out: + // + // %rsp grows (i.e. gets lower) left to right. Each box below is + // one word (eight bytes). Obj0 is the stack slot we're trying to + // get to. + // + // ---------------------------------- + // | BP | Obj0 | Obj1 | ... | ObjN | + // ---------------------------------- + // ^ ^ ^ ^ + // A B C E + // + // A is the incoming stack pointer. + // (B - A) is the local area offset (-8 for x86-64) [1] + // (C - A) is the Offset returned by MFI->getObjectOffset for Obj0 [2] + // + // |(E - B)| is the StackSize (absolute value, positive). For a + // stack that grown down, this works out to be (B - E). [3] + // + // E is also the value of %rsp after stack has been set up, and we + // want (C - E) -- the value we can add to %rsp to get to Obj0. Now + // (C - E) == (C - A) - (B - A) + (B - E) + // { Using [1], [2] and [3] above } + // == getObjectOffset - LocalAreaOffset + StackSize + // + + // Get the Offset from the StackPointer + int Offset = MFI->getObjectOffset(FI) - getOffsetOfLocalArea(); + + return Offset + StackSize; +} +// Simplified from getFrameIndexReference keeping only StackPointer cases +int X86FrameLowering::getFrameIndexReferenceFromSP(const MachineFunction &MF, int FI, + unsigned &FrameReg) const { + const X86RegisterInfo *RegInfo = + static_cast(MF.getSubtarget().getRegisterInfo()); + + assert(!RegInfo->hasBasePointer(MF) && "we don't handle this case"); + + FrameReg = RegInfo->getStackRegister(); + return getFrameIndexOffsetFromSP(MF, FI); +} + bool X86FrameLowering::assignCalleeSavedSpillSlots( MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector &CSI) const { diff --git a/lib/Target/X86/X86FrameLowering.h b/lib/Target/X86/X86FrameLowering.h index 7740c3ad6f3..2ee71159c19 100644 --- a/lib/Target/X86/X86FrameLowering.h +++ b/lib/Target/X86/X86FrameLowering.h @@ -69,6 +69,10 @@ public: int getFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg) const override; + int getFrameIndexOffsetFromSP(const MachineFunction &MF, int FI) const; + int getFrameIndexReferenceFromSP(const MachineFunction &MF, int FI, + unsigned &FrameReg) const override; + void eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override; diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index b67bbcb39c3..7faa58cf377 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -21346,6 +21346,11 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, case X86::EH_SjLj_LongJmp64: return emitEHSjLjLongJmp(MI, BB); + case TargetOpcode::STATEPOINT: + // As an implementation detail, STATEPOINT shares the STACKMAP format at + // this point in the process. We diverge later. + return emitPatchPoint(MI, BB); + case TargetOpcode::STACKMAP: case TargetOpcode::PATCHPOINT: return emitPatchPoint(MI, BB); diff --git a/lib/Target/X86/X86MCInstLower.cpp b/lib/Target/X86/X86MCInstLower.cpp index 4e0d594238c..6a10916aa60 100644 --- a/lib/Target/X86/X86MCInstLower.cpp +++ b/lib/Target/X86/X86MCInstLower.cpp @@ -808,6 +808,66 @@ static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, const MCSu } // while (NumBytes) } +static void LowerSTATEPOINT(MCStreamer &OS, StackMaps &SM, + const MachineInstr &MI, bool Is64Bit, + const TargetMachine& TM, + const MCSubtargetInfo& STI, + X86MCInstLower &MCInstLowering) { + assert(Is64Bit && "Statepoint currently only supports X86-64"); + + // We need to record the frame size for stack walking + const MachineFunction* MF = MI.getParent()->getParent(); + assert(MF && "can't find machine function?"); + + // + // Emit call instruction + // + + // Lower call target and choose correct opcode + const MachineOperand &call_target = StatepointOpers(&MI).getCallTarget(); + MCOperand call_target_mcop; + unsigned call_opcode; + switch (call_target.getType()) { + case MachineOperand::MO_GlobalAddress: + case MachineOperand::MO_ExternalSymbol: + call_target_mcop = MCInstLowering.LowerSymbolOperand( + call_target, + MCInstLowering.GetSymbolFromOperand(call_target)); + call_opcode = X86::CALL64pcrel32; + // Currently, we only support relative addressing with statepoints. + // Otherwise, we'll need a scratch register to hold the target + // address. You'll fail asserts during load & relocation if this + // symbol is to far away. (TODO: support non-relative addressing) + break; + case MachineOperand::MO_Immediate: + call_target_mcop = MCOperand::CreateImm(call_target.getImm()); + call_opcode = X86::CALL64pcrel32; + // Currently, we only support relative addressing with statepoints. + // Otherwise, we'll need a scratch register to hold the target + // immediate. You'll fail asserts during load & relocation if this + // address is to far away. (TODO: support non-relative addressing) + break; + case MachineOperand::MO_Register: + call_target_mcop = MCOperand::CreateReg(call_target.getReg()); + call_opcode = X86::CALL64r; + break; + default: + llvm_unreachable("Unsupported operand type in statepoint call target"); + break; + } + + // Emit call + MCInst call_inst; + call_inst.setOpcode(call_opcode); + call_inst.addOperand(call_target_mcop); + OS.EmitInstruction(call_inst, STI); + + // Record our statepoint node in the same section used by STACKMAP + // and PATCHPOINT + SM.recordStatepoint(MI); +} + + // Lower a stackmap of the form: // , , ... void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) { @@ -1030,7 +1090,9 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { .addExpr(DotExpr)); return; } - + case TargetOpcode::STATEPOINT: + return LowerSTATEPOINT(OutStreamer, SM, *MI, Subtarget->is64Bit(), TM, + getSubtargetInfo(), MCInstLowering); case TargetOpcode::STACKMAP: return LowerSTACKMAP(*MI); diff --git a/utils/TableGen/CodeGenTarget.cpp b/utils/TableGen/CodeGenTarget.cpp index 87777d46018..5b301d70edb 100644 --- a/utils/TableGen/CodeGenTarget.cpp +++ b/utils/TableGen/CodeGenTarget.cpp @@ -300,6 +300,7 @@ void CodeGenTarget::ComputeInstrsByEnum() const { "IMPLICIT_DEF", "SUBREG_TO_REG", "COPY_TO_REGCLASS", "DBG_VALUE", "REG_SEQUENCE", "COPY", "BUNDLE", "LIFETIME_START", "LIFETIME_END", "STACKMAP", "PATCHPOINT", "LOAD_STACK_GUARD", + "STATEPOINT", nullptr}; const DenseMap &Insts = getInstructions(); for (const char *const *p = FixedInstrs; *p; ++p) {