1 #include "llvm/ADT/Triple.h"
2 #include "llvm/ExecutionEngine/Orc/OrcTargetSupport.h"
3 #include "llvm/Support/Process.h"
7 using namespace llvm::orc;
11 uint64_t executeCompileCallback(JITCompileCallbackManagerBase *JCBM,
12 TargetAddress CallbackID) {
13 return JCBM->executeCompileCallback(CallbackID);
21 const char* OrcX86_64::ResolverBlockName = "orc_resolver_block";
23 void OrcX86_64::insertResolverBlock(
24 Module &M, JITCompileCallbackManagerBase &JCBM) {
26 // Trampoline code-sequence length, used to get trampoline address from return
28 const unsigned X86_64_TrampolineLength = 6;
30 // List of x86-64 GPRs to save. Note - RBP saved separately below.
31 std::array<const char *, 14> GPRs = {{
32 "rax", "rbx", "rcx", "rdx",
33 "rsi", "rdi", "r8", "r9",
34 "r10", "r11", "r12", "r13",
38 // Address of the executeCompileCallback function.
39 uint64_t CallbackAddr =
40 static_cast<uint64_t>(
41 reinterpret_cast<uintptr_t>(executeCompileCallback));
43 std::ostringstream AsmStream;
44 Triple TT(M.getTargetTriple());
46 // Switch to text section.
47 if (TT.getOS() == Triple::Darwin)
48 AsmStream << ".section __TEXT,__text,regular,pure_instructions\n"
49 << ".align 4, 0x90\n";
51 AsmStream << ".text\n"
52 << ".align 16, 0x90\n";
54 // Bake in a pointer to the callback manager immediately before the
55 // start of the resolver function.
56 AsmStream << "jit_callback_manager_addr:\n"
57 << " .quad " << &JCBM << "\n";
59 // Start the resolver function.
60 AsmStream << ResolverBlockName << ":\n"
62 << " movq %rsp, %rbp\n";
65 for (const auto &GPR : GPRs)
66 AsmStream << " pushq %" << GPR << "\n";
68 // Store floating-point state with FXSAVE.
69 // Note: We need to keep the stack 16-byte aligned, so if we've emitted an odd
70 // number of 64-bit pushes so far (GPRs.size() plus 1 for RBP) then add
71 // an extra 64 bits of padding to the FXSave area.
72 unsigned Padding = (GPRs.size() + 1) % 2 ? 8 : 0;
73 unsigned FXSaveSize = 512 + Padding;
74 AsmStream << " subq $" << FXSaveSize << ", %rsp\n"
75 << " fxsave64 (%rsp)\n"
77 // Load callback manager address, compute trampoline address, call JIT.
78 << " lea jit_callback_manager_addr(%rip), %rdi\n"
79 << " movq (%rdi), %rdi\n"
80 << " movq 0x8(%rbp), %rsi\n"
81 << " subq $" << X86_64_TrampolineLength << ", %rsi\n"
82 << " movabsq $" << CallbackAddr << ", %rax\n"
85 // Replace the return to the trampoline with the return address of the
86 // compiled function body.
87 << " movq %rax, 0x8(%rbp)\n"
89 // Restore the floating point state.
90 << " fxrstor64 (%rsp)\n"
91 << " addq $" << FXSaveSize << ", %rsp\n";
93 for (const auto &GPR : make_range(GPRs.rbegin(), GPRs.rend()))
94 AsmStream << " popq %" << GPR << "\n";
96 // Restore original RBP and return to compiled function body.
97 AsmStream << " popq %rbp\n"
100 M.appendModuleInlineAsm(AsmStream.str());
103 OrcX86_64::LabelNameFtor
104 OrcX86_64::insertCompileCallbackTrampolines(Module &M,
105 TargetAddress ResolverBlockAddr,
107 unsigned StartIndex) {
108 const char *ResolverBlockPtrName = "Lorc_resolve_block_addr";
110 std::ostringstream AsmStream;
111 Triple TT(M.getTargetTriple());
113 if (TT.getOS() == Triple::Darwin)
114 AsmStream << ".section __TEXT,__text,regular,pure_instructions\n"
115 << ".align 4, 0x90\n";
117 AsmStream << ".text\n"
118 << ".align 16, 0x90\n";
120 AsmStream << ResolverBlockPtrName << ":\n"
121 << " .quad " << ResolverBlockAddr << "\n";
125 std::ostringstream LabelStream;
126 LabelStream << "orc_jcc_" << (StartIndex + I);
127 return LabelStream.str();
130 for (unsigned I = 0; I < NumCalls; ++I)
131 AsmStream << GetLabelName(I) << ":\n"
132 << " callq *" << ResolverBlockPtrName << "(%rip)\n";
134 M.appendModuleInlineAsm(AsmStream.str());
139 OrcX86_64::IndirectStubsInfo::~IndirectStubsInfo() {
140 sys::Memory::releaseMappedMemory(StubsBlock);
141 sys::Memory::releaseMappedMemory(PtrsBlock);
144 std::error_code OrcX86_64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo,
146 void *InitialPtrVal) {
149 // .section __orc_stubs
152 // .byte 0xC4 ; <- Invalid opcode padding.
159 // .section __orc_ptrs
167 const unsigned StubSize = IndirectStubsInfo::StubSize;
169 // Emit at least MinStubs, rounded up to fill the pages allocated.
170 unsigned PageSize = sys::Process::getPageSize();
171 unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize;
172 unsigned NumStubs = (NumPages * PageSize) / StubSize;
174 // Allocate memory for stubs and pointers in one call.
176 auto InitialBlock = sys::Memory::allocateMappedMemory(2 * NumPages * PageSize,
178 sys::Memory::MF_READ |
179 sys::Memory::MF_WRITE,
185 // Create separate MemoryBlocks representing the stubs and pointers.
186 sys::MemoryBlock StubsBlock(InitialBlock.base(), NumPages * PageSize);
187 sys::MemoryBlock PtrsBlock(static_cast<char*>(InitialBlock.base()) +
189 NumPages * PageSize);
191 // Populate the stubs page stubs and mark it executable.
192 uint64_t *Stub = reinterpret_cast<uint64_t*>(StubsBlock.base());
193 uint64_t PtrOffsetField =
194 static_cast<uint64_t>(NumPages * PageSize - 6) << 16;
195 for (unsigned I = 0; I < NumStubs; ++I)
196 Stub[I] = 0xF1C40000000025ff | PtrOffsetField;
198 if (auto EC = sys::Memory::protectMappedMemory(StubsBlock,
199 sys::Memory::MF_READ |
200 sys::Memory::MF_EXEC))
203 // Initialize all pointers to point at FailureAddress.
204 void **Ptr = reinterpret_cast<void**>(PtrsBlock.base());
205 for (unsigned I = 0; I < NumStubs; ++I)
206 Ptr[I] = InitialPtrVal;
208 StubsInfo.NumStubs = NumStubs;
209 StubsInfo.StubsBlock = std::move(StubsBlock);
210 StubsInfo.PtrsBlock = std::move(PtrsBlock);
212 return std::error_code();
215 } // End namespace orc.
216 } // End namespace llvm.