1 //===-- IndirectionUtils.h - Utilities for adding indirections --*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Contains utilities for adding indirections and breaking up modules.
12 //===----------------------------------------------------------------------===//
14 #ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
15 #define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
17 #include "JITSymbol.h"
18 #include "LambdaResolver.h"
19 #include "llvm/ADT/DenseSet.h"
20 #include "llvm/ExecutionEngine/RuntimeDyld.h"
21 #include "llvm/IR/IRBuilder.h"
22 #include "llvm/IR/Mangler.h"
23 #include "llvm/IR/Module.h"
24 #include "llvm/Transforms/Utils/ValueMapper.h"
25 #include "llvm/Support/Process.h"
31 /// @brief Target-independent base class for compile callback management.
32 class JITCompileCallbackManager {
35 typedef std::function<TargetAddress()> CompileFtor;
37 /// @brief Handle to a newly created compile callback. Can be used to get an
38 /// IR constant representing the address of the trampoline, and to set
39 /// the compile action for the callback.
40 class CompileCallbackInfo {
42 CompileCallbackInfo(TargetAddress Addr, CompileFtor &Compile)
43 : Addr(Addr), Compile(Compile) {}
45 TargetAddress getAddress() const { return Addr; }
46 void setCompileAction(CompileFtor Compile) {
47 this->Compile = std::move(Compile);
54 /// @brief Construct a JITCompileCallbackManager.
55 /// @param ErrorHandlerAddress The address of an error handler in the target
56 /// process to be used if a compile callback fails.
57 JITCompileCallbackManager(TargetAddress ErrorHandlerAddress)
58 : ErrorHandlerAddress(ErrorHandlerAddress) {}
60 virtual ~JITCompileCallbackManager() {}
62 /// @brief Execute the callback for the given trampoline id. Called by the JIT
63 /// to compile functions on demand.
64 TargetAddress executeCompileCallback(TargetAddress TrampolineAddr) {
65 auto I = ActiveTrampolines.find(TrampolineAddr);
66 // FIXME: Also raise an error in the Orc error-handler when we finally have
68 if (I == ActiveTrampolines.end())
69 return ErrorHandlerAddress;
71 // Found a callback handler. Yank this trampoline out of the active list and
72 // put it back in the available trampolines list, then try to run the
73 // handler's compile and update actions.
74 // Moving the trampoline ID back to the available list first means there's at
75 // least one available trampoline if the compile action triggers a request for
77 auto Compile = std::move(I->second);
78 ActiveTrampolines.erase(I);
79 AvailableTrampolines.push_back(TrampolineAddr);
81 if (auto Addr = Compile())
84 return ErrorHandlerAddress;
87 /// @brief Reserve a compile callback.
88 CompileCallbackInfo getCompileCallback() {
89 TargetAddress TrampolineAddr = getAvailableTrampolineAddr();
90 auto &Compile = this->ActiveTrampolines[TrampolineAddr];
91 return CompileCallbackInfo(TrampolineAddr, Compile);
94 /// @brief Get a CompileCallbackInfo for an existing callback.
95 CompileCallbackInfo getCompileCallbackInfo(TargetAddress TrampolineAddr) {
96 auto I = ActiveTrampolines.find(TrampolineAddr);
97 assert(I != ActiveTrampolines.end() && "Not an active trampoline.");
98 return CompileCallbackInfo(I->first, I->second);
101 /// @brief Release a compile callback.
103 /// Note: Callbacks are auto-released after they execute. This method should
104 /// only be called to manually release a callback that is not going to
106 void releaseCompileCallback(TargetAddress TrampolineAddr) {
107 auto I = ActiveTrampolines.find(TrampolineAddr);
108 assert(I != ActiveTrampolines.end() && "Not an active trampoline.");
109 ActiveTrampolines.erase(I);
110 AvailableTrampolines.push_back(TrampolineAddr);
114 TargetAddress ErrorHandlerAddress;
116 typedef std::map<TargetAddress, CompileFtor> TrampolineMapT;
117 TrampolineMapT ActiveTrampolines;
118 std::vector<TargetAddress> AvailableTrampolines;
122 TargetAddress getAvailableTrampolineAddr() {
123 if (this->AvailableTrampolines.empty())
125 assert(!this->AvailableTrampolines.empty() &&
126 "Failed to grow available trampolines.");
127 TargetAddress TrampolineAddr = this->AvailableTrampolines.back();
128 this->AvailableTrampolines.pop_back();
129 return TrampolineAddr;
132 // Create new trampolines - to be implemented in subclasses.
133 virtual void grow() = 0;
135 virtual void anchor();
138 /// @brief Manage compile callbacks for in-process JITs.
139 template <typename TargetT>
140 class LocalJITCompileCallbackManager : public JITCompileCallbackManager {
143 /// @brief Construct a InProcessJITCompileCallbackManager.
144 /// @param ErrorHandlerAddress The address of an error handler in the target
145 /// process to be used if a compile callback fails.
146 LocalJITCompileCallbackManager(TargetAddress ErrorHandlerAddress)
147 : JITCompileCallbackManager(ErrorHandlerAddress) {
149 /// Set up the resolver block.
152 sys::OwningMemoryBlock(
153 sys::Memory::allocateMappedMemory(TargetT::ResolverCodeSize, nullptr,
154 sys::Memory::MF_READ |
155 sys::Memory::MF_WRITE, EC));
156 assert(!EC && "Failed to allocate resolver block");
158 TargetT::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()),
161 EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),
162 sys::Memory::MF_READ |
163 sys::Memory::MF_EXEC);
164 assert(!EC && "Failed to mprotect resolver block");
169 static TargetAddress reenter(void *CCMgr, void *TrampolineId) {
170 JITCompileCallbackManager *Mgr =
171 static_cast<JITCompileCallbackManager*>(CCMgr);
172 return Mgr->executeCompileCallback(
173 static_cast<TargetAddress>(
174 reinterpret_cast<uintptr_t>(TrampolineId)));
177 void grow() override {
178 assert(this->AvailableTrampolines.empty() && "Growing prematurely?");
181 auto TrampolineBlock =
182 sys::OwningMemoryBlock(
183 sys::Memory::allocateMappedMemory(sys::Process::getPageSize(), nullptr,
184 sys::Memory::MF_READ |
185 sys::Memory::MF_WRITE, EC));
186 assert(!EC && "Failed to allocate trampoline block");
189 unsigned NumTrampolines =
190 (sys::Process::getPageSize() - TargetT::PointerSize) /
191 TargetT::TrampolineSize;
193 uint8_t *TrampolineMem = static_cast<uint8_t*>(TrampolineBlock.base());
194 TargetT::writeTrampolines(TrampolineMem, ResolverBlock.base(),
197 for (unsigned I = 0; I < NumTrampolines; ++I)
198 this->AvailableTrampolines.push_back(
199 static_cast<TargetAddress>(reinterpret_cast<uintptr_t>(
200 TrampolineMem + (I * TargetT::TrampolineSize))));
202 EC = sys::Memory::protectMappedMemory(TrampolineBlock.getMemoryBlock(),
203 sys::Memory::MF_READ |
204 sys::Memory::MF_EXEC);
205 assert(!EC && "Failed to mprotect trampoline block");
207 TrampolineBlocks.push_back(std::move(TrampolineBlock));
210 sys::OwningMemoryBlock ResolverBlock;
211 std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
214 /// @brief Base class for managing collections of named indirect stubs.
215 class IndirectStubsManager {
218 /// @brief Map type for initializing the manager. See init.
219 typedef StringMap<std::pair<TargetAddress, JITSymbolFlags>> StubInitsMap;
221 virtual ~IndirectStubsManager() {}
223 /// @brief Create a single stub with the given name, target address and flags.
224 virtual std::error_code createStub(StringRef StubName, TargetAddress StubAddr,
225 JITSymbolFlags StubFlags) = 0;
227 /// @brief Create StubInits.size() stubs with the given names, target
228 /// addresses, and flags.
229 virtual std::error_code createStubs(const StubInitsMap &StubInits) = 0;
231 /// @brief Find the stub with the given name. If ExportedStubsOnly is true,
232 /// this will only return a result if the stub's flags indicate that it
234 virtual JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0;
236 /// @brief Find the implementation-pointer for the stub.
237 virtual JITSymbol findPointer(StringRef Name) = 0;
239 /// @brief Change the value of the implementation pointer for the stub.
240 virtual std::error_code updatePointer(StringRef Name, TargetAddress NewAddr) = 0;
242 virtual void anchor();
245 /// @brief IndirectStubsManager implementation for the host architecture, e.g.
246 /// OrcX86_64. (See OrcArchitectureSupport.h).
247 template <typename TargetT>
248 class LocalIndirectStubsManager : public IndirectStubsManager {
251 std::error_code createStub(StringRef StubName, TargetAddress StubAddr,
252 JITSymbolFlags StubFlags) override {
253 if (auto EC = reserveStubs(1))
256 createStubInternal(StubName, StubAddr, StubFlags);
258 return std::error_code();
261 std::error_code createStubs(const StubInitsMap &StubInits) override {
262 if (auto EC = reserveStubs(StubInits.size()))
265 for (auto &Entry : StubInits)
266 createStubInternal(Entry.first(), Entry.second.first,
267 Entry.second.second);
269 return std::error_code();
272 JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) override {
273 auto I = StubIndexes.find(Name);
274 if (I == StubIndexes.end())
276 auto Key = I->second.first;
277 void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);
278 assert(StubAddr && "Missing stub address");
279 auto StubTargetAddr =
280 static_cast<TargetAddress>(reinterpret_cast<uintptr_t>(StubAddr));
281 auto StubSymbol = JITSymbol(StubTargetAddr, I->second.second);
282 if (ExportedStubsOnly && !StubSymbol.isExported())
287 JITSymbol findPointer(StringRef Name) override {
288 auto I = StubIndexes.find(Name);
289 if (I == StubIndexes.end())
291 auto Key = I->second.first;
292 void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);
293 assert(PtrAddr && "Missing pointer address");
295 static_cast<TargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr));
296 return JITSymbol(PtrTargetAddr, I->second.second);
299 std::error_code updatePointer(StringRef Name, TargetAddress NewAddr) override {
300 auto I = StubIndexes.find(Name);
301 assert(I != StubIndexes.end() && "No stub pointer for symbol");
302 auto Key = I->second.first;
303 *IndirectStubsInfos[Key.first].getPtr(Key.second) =
304 reinterpret_cast<void*>(static_cast<uintptr_t>(NewAddr));
305 return std::error_code();
310 std::error_code reserveStubs(unsigned NumStubs) {
311 if (NumStubs <= FreeStubs.size())
312 return std::error_code();
314 unsigned NewStubsRequired = NumStubs - FreeStubs.size();
315 unsigned NewBlockId = IndirectStubsInfos.size();
316 typename TargetT::IndirectStubsInfo ISI;
317 if (auto EC = TargetT::emitIndirectStubsBlock(ISI, NewStubsRequired,
320 for (unsigned I = 0; I < ISI.getNumStubs(); ++I)
321 FreeStubs.push_back(std::make_pair(NewBlockId, I));
322 IndirectStubsInfos.push_back(std::move(ISI));
323 return std::error_code();
326 void createStubInternal(StringRef StubName, TargetAddress InitAddr,
327 JITSymbolFlags StubFlags) {
328 auto Key = FreeStubs.back();
329 FreeStubs.pop_back();
330 *IndirectStubsInfos[Key.first].getPtr(Key.second) =
331 reinterpret_cast<void*>(static_cast<uintptr_t>(InitAddr));
332 StubIndexes[StubName] = std::make_pair(Key, StubFlags);
335 std::vector<typename TargetT::IndirectStubsInfo> IndirectStubsInfos;
336 typedef std::pair<uint16_t, uint16_t> StubKey;
337 std::vector<StubKey> FreeStubs;
338 StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;
341 /// @brief Build a function pointer of FunctionType with the given constant
344 /// Usage example: Turn a trampoline address into a function pointer constant
345 /// for use in a stub.
346 Constant* createIRTypedAddress(FunctionType &FT, TargetAddress Addr);
348 /// @brief Create a function pointer with the given type, name, and initializer
349 /// in the given Module.
350 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
351 const Twine &Name, Constant *Initializer);
353 /// @brief Turn a function declaration into a stub function that makes an
354 /// indirect call using the given function pointer.
355 void makeStub(Function &F, Value &ImplPointer);
357 /// @brief Raise linkage types and rename as necessary to ensure that all
358 /// symbols are accessible for other modules.
360 /// This should be called before partitioning a module to ensure that the
361 /// partitions retain access to each other's symbols.
362 void makeAllSymbolsExternallyAccessible(Module &M);
364 /// @brief Clone a function declaration into a new module.
366 /// This function can be used as the first step towards creating a callback
367 /// stub (see makeStub), or moving a function body (see moveFunctionBody).
369 /// If the VMap argument is non-null, a mapping will be added between F and
370 /// the new declaration, and between each of F's arguments and the new
371 /// declaration's arguments. This map can then be passed in to moveFunction to
372 /// move the function body if required. Note: When moving functions between
373 /// modules with these utilities, all decls should be cloned (and added to a
374 /// single VMap) before any bodies are moved. This will ensure that references
375 /// between functions all refer to the versions in the new module.
376 Function* cloneFunctionDecl(Module &Dst, const Function &F,
377 ValueToValueMapTy *VMap = nullptr);
379 /// @brief Move the body of function 'F' to a cloned function declaration in a
380 /// different module (See related cloneFunctionDecl).
382 /// If the target function declaration is not supplied via the NewF parameter
383 /// then it will be looked up via the VMap.
385 /// This will delete the body of function 'F' from its original parent module,
386 /// but leave its declaration.
387 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
388 ValueMaterializer *Materializer = nullptr,
389 Function *NewF = nullptr);
391 /// @brief Clone a global variable declaration into a new module.
392 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
393 ValueToValueMapTy *VMap = nullptr);
395 /// @brief Move global variable GV from its parent module to cloned global
396 /// declaration in a different module.
398 /// If the target global declaration is not supplied via the NewGV parameter
399 /// then it will be looked up via the VMap.
401 /// This will delete the initializer of GV from its original parent module,
402 /// but leave its declaration.
403 void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
404 ValueToValueMapTy &VMap,
405 ValueMaterializer *Materializer = nullptr,
406 GlobalVariable *NewGV = nullptr);
409 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
410 ValueToValueMapTy &VMap);
412 } // End namespace orc.
413 } // End namespace llvm.
415 #endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H