From 0d40b2b30f41b90f5056bfa83c98c10fb4227a07 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Tue, 17 Feb 2015 01:18:38 +0000 Subject: [PATCH] [Orc] Update the Orc indirection utils and refactor the CompileOnDemand layer. This patch replaces most of the Orc indirection utils API with a new class: JITCompileCallbackManager, which creates and manages JIT callbacks. Exposing this functionality directly allows the user to create callbacks that are associated with user supplied compilation actions. For example, you can create a callback to lazyily IR-gen something from an AST. (A kaleidoscope example demonstrating this will be committed shortly). This patch also refactors the CompileOnDemand layer to use the JITCompileCallbackManager API. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@229461 91177308-0d34-0410-b5e6-96231b3b80d8 --- .../llvm/ExecutionEngine/Orc/CloneSubModule.h | 7 +- .../Orc/CompileOnDemandLayer.h | 247 ++++++---- .../ExecutionEngine/Orc/IndirectionUtils.h | 429 ++++++++---------- .../ExecutionEngine/Orc/OrcTargetSupport.h | 34 +- lib/ExecutionEngine/Orc/CloneSubModule.cpp | 40 +- lib/ExecutionEngine/Orc/IndirectionUtils.cpp | 200 ++++---- lib/ExecutionEngine/Orc/OrcTargetSupport.cpp | 114 +++-- 7 files changed, 552 insertions(+), 519 deletions(-) diff --git a/include/llvm/ExecutionEngine/Orc/CloneSubModule.h b/include/llvm/ExecutionEngine/Orc/CloneSubModule.h index 3e2f1b3cd92..208a4f7692b 100644 --- a/include/llvm/ExecutionEngine/Orc/CloneSubModule.h +++ b/include/llvm/ExecutionEngine/Orc/CloneSubModule.h @@ -15,6 +15,7 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_CLONESUBMODULE_H #define LLVM_EXECUTIONENGINE_ORC_CLONESUBMODULE_H +#include "llvm/ADT/DenseSet.h" #include "llvm/Transforms/Utils/ValueMapper.h" #include @@ -36,9 +37,9 @@ void copyGVInitializer(GlobalVariable &New, const GlobalVariable &Orig, void copyFunctionBody(Function &New, const Function &Orig, ValueToValueMapTy &VMap); -std::unique_ptr -CloneSubModule(const Module &M, HandleGlobalVariableFtor HandleGlobalVariable, - HandleFunctionFtor HandleFunction, bool KeepInlineAsm); +void CloneSubModule(Module &Dst, const Module &Src, + HandleGlobalVariableFtor HandleGlobalVariable, + HandleFunctionFtor HandleFunction, bool KeepInlineAsm); } #endif // LLVM_EXECUTIONENGINE_ORC_CLONESUBMODULE_H diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 25381f1bdd8..7fb25db0322 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -33,7 +33,8 @@ namespace llvm { /// It is expected that this layer will frequently be used on top of a /// LazyEmittingLayer. The combination of the two ensures that each function is /// compiled only when it is first called. -template class CompileOnDemandLayer { +template +class CompileOnDemandLayer { public: /// @brief Lookup helper that provides compatibility with the classic /// static-compilation symbol resolution process. @@ -114,13 +115,6 @@ private: // Logical module handles. std::vector LMHandles; - // Persistent manglers - one per TU. - std::vector PersistentManglers; - - // Symbol resolution callback handlers - one per TU. - std::vector> - JITResolveCallbackHandlers; - // List of vectors of module set handles: // One vector per logical module - each vector holds the handles for the // exploded modules for that logical module in the base layer. @@ -143,91 +137,37 @@ public: /// @brief Handle to a set of loaded modules. typedef typename ModuleSetInfoListT::iterator ModuleSetHandleT; - /// @brief Convenience typedef for callback inserter. - typedef std::function - InsertCallbackAsmFtor; + // @brief Fallback lookup functor. + typedef std::function LookupFtor; /// @brief Construct a compile-on-demand layer instance. - CompileOnDemandLayer(BaseLayerT &BaseLayer, - InsertCallbackAsmFtor InsertCallbackAsm) - : BaseLayer(BaseLayer), InsertCallbackAsm(InsertCallbackAsm) {} + CompileOnDemandLayer(BaseLayerT &BaseLayer, LLVMContext &Context) + : BaseLayer(BaseLayer), + CompileCallbackMgr(BaseLayer, Context, 0, 64) {} /// @brief Add a module to the compile-on-demand layer. template ModuleSetHandleT addModuleSet(ModuleSetT Ms, - std::unique_ptr MM) { - - const char *JITAddrSuffix = "$orc_addr"; - const char *JITImplSuffix = "$orc_impl"; - - // Create a symbol lookup context and ModuleSetInfo for this module set. + LookupFtor FallbackLookup = nullptr) { + + // If the user didn't supply a fallback lookup then just use + // getSymbolAddress. + if (!FallbackLookup) + FallbackLookup = [=](const std::string &Name) { + return findSymbol(Name, true).getAddress(); + }; + + // Create a lookup context and ModuleSetInfo for this module set. + // For the purposes of symbol resolution the set Ms will be treated as if + // the modules it contained had been linked together as a dylib. auto DylibLookup = std::make_shared(BaseLayer); ModuleSetHandleT H = ModuleSetInfos.insert(ModuleSetInfos.end(), ModuleSetInfo(DylibLookup)); ModuleSetInfo &MSI = ModuleSetInfos.back(); - // Process each of the modules in this module set. All modules share the - // same lookup context, but each will get its own TU lookup context. - for (auto &M : Ms) { - - // Create a TU lookup context for this module. - auto LMH = DylibLookup->createLogicalModule(); - MSI.LMHandles.push_back(LMH); - - // Create a persistent mangler for this module. - MSI.PersistentManglers.emplace_back(*M->getDataLayout()); - - // Make all calls to functions defined in this module indirect. - JITIndirections Indirections = - makeCallsDoubleIndirect(*M, [](const Function &) { return true; }, - JITImplSuffix, JITAddrSuffix); - - // Then carve up the module into a bunch of single-function modules. - std::vector> ExplodedModules = - explode(*M, Indirections); - - // Add a resolve-callback handler for this module to look up symbol - // addresses when requested via a callback. - MSI.JITResolveCallbackHandlers.push_back( - createCallbackHandlerFromJITIndirections( - Indirections, MSI.PersistentManglers.back(), - [=](StringRef S) { - return DylibLookup->findSymbol(LMH, S).getAddress(); - })); - - // Insert callback asm code into the first module. - InsertCallbackAsm(*ExplodedModules[0], - *MSI.JITResolveCallbackHandlers.back()); - - // Now we need to take each of the extracted Modules and add them to - // base layer. Each Module will be added individually to make sure they - // can be compiled separately, and each will get its own lookaside - // memory manager with lookup functors that resolve symbols in sibling - // modules first.OA - for (auto &M : ExplodedModules) { - std::vector> MSet; - MSet.push_back(std::move(M)); - - BaseLayerModuleSetHandleT H = BaseLayer.addModuleSet( - std::move(MSet), - createLookasideRTDyldMM( - [=](const std::string &Name) { - if (auto Symbol = DylibLookup->findSymbol(LMH, Name)) - return Symbol.getAddress(); - return findSymbol(Name, true).getAddress(); - }, - [=](const std::string &Name) { - return DylibLookup->findSymbol(LMH, Name).getAddress(); - })); - DylibLookup->addToLogicalModule(LMH, H); - MSI.BaseLayerModuleSetHandles.push_back(H); - } - - initializeFuncAddrs(*MSI.JITResolveCallbackHandlers.back(), Indirections, - MSI.PersistentManglers.back(), [=](StringRef S) { - return DylibLookup->findSymbol(LMH, S).getAddress(); - }); - } + // Process each of the modules in this module set. + for (auto &M : Ms) + partitionAndAdd(*M, MSI, FallbackLookup); return H; } @@ -262,8 +202,149 @@ public: } private: + + void partitionAndAdd(Module &M, ModuleSetInfo &MSI, + LookupFtor FallbackLookup) { + const char *AddrSuffix = "$orc_addr"; + const char *BodySuffix = "$orc_body"; + + // We're going to break M up into a bunch of sub-modules, but we want + // internal linkage symbols to still resolve sensibly. CODScopedLookup + // provides the "logical module" concept to make this work, so create a + // new logical module for M. + auto DylibLookup = MSI.Lookup; + auto LogicalModule = DylibLookup->createLogicalModule(); + MSI.LMHandles.push_back(LogicalModule); + + // Partition M into a "globals and stubs" module, a "common symbols" module, + // and a list of single-function modules. + auto PartitionedModule = fullyPartition(M); + auto StubsModule = std::move(PartitionedModule.GlobalVars); + auto CommonsModule = std::move(PartitionedModule.Commons); + auto FunctionModules = std::move(PartitionedModule.Functions); + + // Emit the commons stright away. + auto CommonHandle = addModule(std::move(CommonsModule), MSI, LogicalModule, + FallbackLookup); + BaseLayer.emitAndFinalize(CommonHandle); + + // Map of definition names to callback-info data structures. We'll use + // this to build the compile actions for the stubs below. + typedef std::map + StubInfoMap; + StubInfoMap StubInfos; + + // Now we need to take each of the extracted Modules and add them to + // base layer. Each Module will be added individually to make sure they + // can be compiled separately, and each will get its own lookaside + // memory manager that will resolve within this logical module first. + for (auto &SubM : FunctionModules) { + + // Keep track of the stubs we create for this module so that we can set + // their compile actions. + std::vector NewStubInfos; + + // Search for function definitions and insert stubs into the stubs + // module. + for (auto &F : *SubM) { + if (F.isDeclaration()) + continue; + + std::string Name = F.getName(); + Function *Proto = StubsModule->getFunction(Name); + assert(Proto && "Failed to clone function decl into stubs module."); + auto CallbackInfo = + CompileCallbackMgr.getCompileCallback(*Proto->getFunctionType()); + GlobalVariable *FunctionBodyPointer = + createImplPointer(*Proto, Name + AddrSuffix, + CallbackInfo.getAddress()); + makeStub(*Proto, *FunctionBodyPointer); + + F.setName(Name + BodySuffix); + F.setVisibility(GlobalValue::HiddenVisibility); + + auto KV = std::make_pair(std::move(Name), std::move(CallbackInfo)); + NewStubInfos.push_back(StubInfos.insert(StubInfos.begin(), KV)); + } + + auto H = addModule(std::move(SubM), MSI, LogicalModule, FallbackLookup); + + // Set the compile actions for this module: + for (auto &KVPair : NewStubInfos) { + std::string BodyName = Mangle(KVPair->first + BodySuffix, + *M.getDataLayout()); + auto &CCInfo = KVPair->second; + CCInfo.setCompileAction( + [=](){ + return BaseLayer.findSymbolIn(H, BodyName, false).getAddress(); + }); + } + + } + + // Ok - we've processed all the partitioned modules. Now add the + // stubs/globals module and set the update actions. + auto StubsH = + addModule(std::move(StubsModule), MSI, LogicalModule, FallbackLookup); + + for (auto &KVPair : StubInfos) { + std::string AddrName = Mangle(KVPair.first + AddrSuffix, + *M.getDataLayout()); + auto &CCInfo = KVPair.second; + CCInfo.setUpdateAction( + CompileCallbackMgr.getLocalFPUpdater(StubsH, AddrName)); + } + } + + // Add the given Module to the base layer using a memory manager that will + // perform the appropriate scoped lookup (i.e. will look first with in the + // module from which it was extracted, then into the set to which that module + // belonged, and finally externally). + BaseLayerModuleSetHandleT addModule( + std::unique_ptr M, + ModuleSetInfo &MSI, + typename CODScopedLookup::LMHandle LogicalModule, + LookupFtor FallbackLookup) { + + // Add this module to the JIT with a memory manager that uses the + // DylibLookup to resolve symbols. + std::vector> MSet; + MSet.push_back(std::move(M)); + + auto DylibLookup = MSI.Lookup; + auto MM = + createLookasideRTDyldMM( + [=](const std::string &Name) { + if (auto Symbol = DylibLookup->findSymbol(LogicalModule, Name)) + return Symbol.getAddress(); + return FallbackLookup(Name); + }, + [=](const std::string &Name) { + return DylibLookup->findSymbol(LogicalModule, Name).getAddress(); + }); + + BaseLayerModuleSetHandleT H = + BaseLayer.addModuleSet(std::move(MSet), std::move(MM)); + // Add this module to the logical module lookup. + DylibLookup->addToLogicalModule(LogicalModule, H); + MSI.BaseLayerModuleSetHandles.push_back(H); + + return H; + } + + static std::string Mangle(StringRef Name, const DataLayout &DL) { + Mangler M(&DL); + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + M.getNameWithPrefix(MangledNameStream, Name); + } + return MangledName; + } + BaseLayerT &BaseLayer; - InsertCallbackAsmFtor InsertCallbackAsm; + CompileCallbackMgrT CompileCallbackMgr; ModuleSetInfoListT ModuleSetInfos; }; } diff --git a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index 0bc71bfdf33..72cf335cac7 100644 --- a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -15,271 +15,224 @@ #define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H #include "JITSymbol.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" #include namespace llvm { -/// @brief Persistent name mangling. -/// -/// This class provides name mangling that can outlive a Module (and its -/// DataLayout). -class PersistentMangler { +/// @brief Base class for JITLayer independent aspects of +/// JITCompileCallbackManager. +template +class JITCompileCallbackManagerBase { public: - PersistentMangler(DataLayout DL) : DL(std::move(DL)), M(&this->DL) {} - std::string getMangledName(StringRef Name) const { - std::string MangledName; - { - raw_string_ostream MangledNameStream(MangledName); - M.getNameWithPrefix(MangledNameStream, Name); + /// @brief Construct a JITCompileCallbackManagerBase. + /// @param ErrorHandlerAddress The address of an error handler in the target + /// process to be used if a compile callback fails. + /// @param NumTrampolinesPerBlock Number of trampolines to emit if there is no + /// available trampoline when getCompileCallback is + /// called. + JITCompileCallbackManagerBase(TargetAddress ErrorHandlerAddress, + unsigned NumTrampolinesPerBlock) + : ErrorHandlerAddress(ErrorHandlerAddress), + NumTrampolinesPerBlock(NumTrampolinesPerBlock) {} + + /// @brief Execute the callback for the given trampoline id. Called by the JIT + /// to compile functions on demand. + TargetAddress executeCompileCallback(TargetAddress TrampolineID) { + typename TrampolineMapT::iterator I = ActiveTrampolines.find(TrampolineID); + // FIXME: Also raise an error in the Orc error-handler when we finally have + // one. + if (I == ActiveTrampolines.end()) + return ErrorHandlerAddress; + + // Found a callback handler. Yank this trampoline out of the active list and + // put it back in the available trampolines list, then try to run the + // handler's compile and update actions. + // Moving the trampoline ID back to the available list first means there's at + // least one available trampoline if the compile action triggers a request for + // a new one. + AvailableTrampolines.push_back(I->first); + auto CallbackHandler = std::move(I->second); + ActiveTrampolines.erase(I); + + if (auto Addr = CallbackHandler.Compile()) { + CallbackHandler.Update(Addr); + return Addr; } - return MangledName; + return ErrorHandlerAddress; } -private: - DataLayout DL; - Mangler M; -}; +protected: -/// @brief Handle callbacks from the JIT process requesting the definitions of -/// symbols. -/// -/// This utility is intended to be used to support compile-on-demand for -/// functions. -class JITResolveCallbackHandler { -private: - typedef std::vector FuncNameList; + typedef std::function CompileFtorT; + typedef std::function UpdateFtorT; -public: - typedef FuncNameList::size_type StubIndex; + struct CallbackHandler { + CompileFtorT Compile; + UpdateFtorT Update; + }; -public: - /// @brief Create a JITResolveCallbackHandler with the given functors for - /// looking up symbols and updating their use-sites. - /// - /// @return A JITResolveCallbackHandler instance that will invoke the - /// Lookup and Update functors as needed to resolve missing symbol - /// definitions. - template - static std::unique_ptr create(LookupFtor Lookup, - UpdateFtor Update); - - /// @brief Destroy instance. Does not modify existing emitted symbols. - /// - /// Not-yet-emitted symbols will need to be resolved some other way after - /// this class is destroyed. - virtual ~JITResolveCallbackHandler() {} - - /// @brief Add a function to be resolved on demand. - void addFuncName(std::string Name) { FuncNames.push_back(std::move(Name)); } - - /// @brief Get the name associated with the given index. - const std::string &getFuncName(StubIndex Idx) const { return FuncNames[Idx]; } - - /// @brief Returns the number of symbols being managed by this instance. - StubIndex getNumFuncs() const { return FuncNames.size(); } - - /// @brief Get the address for the symbol associated with the given index. - /// - /// This is expected to be called by code in the JIT process itself, in - /// order to resolve a function. - virtual TargetAddress resolve(StubIndex StubIdx) = 0; + TargetAddress ErrorHandlerAddress; + unsigned NumTrampolinesPerBlock; -private: - FuncNameList FuncNames; + typedef std::map TrampolineMapT; + TrampolineMapT ActiveTrampolines; + std::vector AvailableTrampolines; }; -// Implementation class for JITResolveCallbackHandler. -template -class JITResolveCallbackHandlerImpl : public JITResolveCallbackHandler { +/// @brief Manage compile callbacks. +template +class JITCompileCallbackManager : + public JITCompileCallbackManagerBase { public: - JITResolveCallbackHandlerImpl(LookupFtor Lookup, UpdateFtor Update) - : Lookup(std::move(Lookup)), Update(std::move(Update)) {} - - TargetAddress resolve(StubIndex StubIdx) override { - const std::string &FuncName = getFuncName(StubIdx); - TargetAddress Addr = Lookup(FuncName); - Update(FuncName, Addr); - return Addr; + + typedef typename JITCompileCallbackManagerBase::CompileFtorT + CompileFtorT; + typedef typename JITCompileCallbackManagerBase::UpdateFtorT + UpdateFtorT; + + /// @brief Construct a JITCompileCallbackManager. + /// @param JIT JIT layer to emit callback trampolines, etc. into. + /// @param Context LLVMContext to use for trampoline & resolve block modules. + /// @param ErrorHandlerAddress The address of an error handler in the target + /// process to be used if a compile callback fails. + /// @param NumTrampolinesPerBlock Number of trampolines to allocate whenever + /// there is no existing callback trampoline. + /// (Trampolines are allocated in blocks for + /// efficiency.) + JITCompileCallbackManager(JITLayerT &JIT, LLVMContext &Context, + TargetAddress ErrorHandlerAddress, + unsigned NumTrampolinesPerBlock) + : JITCompileCallbackManagerBase(ErrorHandlerAddress, + NumTrampolinesPerBlock), + JIT(JIT) { + emitResolverBlock(Context); + } + + /// @brief Handle to a newly created compile callback. Can be used to get an + /// IR constant representing the address of the trampoline, and to set + /// the compile and update actions for the callback. + class CompileCallbackInfo { + public: + CompileCallbackInfo(Constant *Addr, CompileFtorT &Compile, + UpdateFtorT &Update) + : Addr(Addr), Compile(Compile), Update(Update) {} + + Constant* getAddress() const { return Addr; } + void setCompileAction(CompileFtorT Compile) { + this->Compile = std::move(Compile); + } + void setUpdateAction(UpdateFtorT Update) { + this->Update = std::move(Update); + } + private: + Constant *Addr; + CompileFtorT &Compile; + UpdateFtorT &Update; + }; + + /// @brief Get/create a compile callback with the given signature. + CompileCallbackInfo getCompileCallback(FunctionType &FT) { + TargetAddress TrampolineAddr = getAvailableTrampolineAddr(FT.getContext()); + auto &CallbackHandler = + this->ActiveTrampolines[TrampolineAddr + TargetT::CallSize]; + Constant *AddrIntVal = + ConstantInt::get(Type::getInt64Ty(FT.getContext()), TrampolineAddr); + Constant *AddrPtrVal = + ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, + PointerType::get(&FT, 0)); + + return CompileCallbackInfo(AddrPtrVal, CallbackHandler.Compile, + CallbackHandler.Update); + } + + /// @brief Get a functor for updating the value of a named function pointer. + UpdateFtorT getLocalFPUpdater(typename JITLayerT::ModuleSetHandleT H, + std::string Name) { + // FIXME: Move-capture Name once we can use C++14. + return [=](TargetAddress Addr) { + auto FPSym = JIT.findSymbolIn(H, Name, true); + assert(FPSym && "Cannot find function pointer to update."); + void *FPAddr = reinterpret_cast( + static_cast(FPSym.getAddress())); + memcpy(FPAddr, &Addr, sizeof(uintptr_t)); + }; } private: - LookupFtor Lookup; - UpdateFtor Update; -}; -template -std::unique_ptr -JITResolveCallbackHandler::create(LookupFtor Lookup, UpdateFtor Update) { - typedef JITResolveCallbackHandlerImpl Impl; - return make_unique(std::move(Lookup), std::move(Update)); -} + std::vector> + SingletonSet(std::unique_ptr M) { + std::vector> Ms; + Ms.push_back(std::move(M)); + return Ms; + } -/// @brief Holds a list of the function names that were indirected, plus -/// mappings from each of these names to (a) the name of function -/// providing the implementation for that name (GetImplNames), and -/// (b) the name of the global variable holding the address of the -/// implementation. -/// -/// This data structure can be used with a JITCallbackHandler to look up and -/// update function implementations when lazily compiling. -class JITIndirections { -public: - JITIndirections(std::vector IndirectedNames, - std::function GetImplName, - std::function GetAddrName) - : IndirectedNames(std::move(IndirectedNames)), - GetImplName(std::move(GetImplName)), - GetAddrName(std::move(GetAddrName)) {} - - std::vector IndirectedNames; - std::function GetImplName; - std::function GetAddrName; -}; + void emitResolverBlock(LLVMContext &Context) { + std::unique_ptr M(new Module("resolver_block_module", + Context)); + TargetT::insertResolverBlock(*M, *this); + auto H = JIT.addModuleSet(SingletonSet(std::move(M)), nullptr); + JIT.emitAndFinalize(H); + auto ResolverBlockSymbol = + JIT.findSymbolIn(H, TargetT::ResolverBlockName, false); + assert(ResolverBlockSymbol && "Failed to insert resolver block"); + ResolverBlockAddr = ResolverBlockSymbol.getAddress(); + } -/// @brief Indirect all calls to functions matching the predicate -/// ShouldIndirect through a global variable containing the address -/// of the implementation. -/// -/// @return An indirection structure containing the functions that had their -/// call-sites re-written. -/// -/// For each function 'F' that meets the ShouldIndirect predicate, and that -/// is called in this Module, add a common-linkage global variable to the -/// module that will hold the address of the implementation of that function. -/// Rewrite all call-sites of 'F' to be indirect calls (via the global). -/// This allows clients, either directly or via a JITCallbackHandler, to -/// change the address of the implementation of 'F' at runtime. -/// -/// Important notes: -/// -/// Single indirection does not preserve pointer equality for 'F'. If the -/// program was already calling 'F' indirectly through function pointers, or -/// if it was taking the address of 'F' for the purpose of pointer comparisons -/// or arithmetic double indirection should be used instead. -/// -/// This method does *not* initialize the function implementation addresses. -/// The client must do this prior to running any call-sites that have been -/// indirected. -JITIndirections makeCallsSingleIndirect( - llvm::Module &M, - const std::function &ShouldIndirect, - const char *JITImplSuffix, const char *JITAddrSuffix); - -/// @brief Replace the body of functions matching the predicate ShouldIndirect -/// with indirect calls to the implementation. -/// -/// @return An indirections structure containing the functions that had their -/// implementations re-written. -/// -/// For each function 'F' that meets the ShouldIndirect predicate, add a -/// common-linkage global variable to the module that will hold the address of -/// the implementation of that function and rewrite the implementation of 'F' -/// to call through to the implementation indirectly (via the global). -/// This allows clients, either directly or via a JITCallbackHandler, to -/// change the address of the implementation of 'F' at runtime. -/// -/// Important notes: -/// -/// Double indirection is slower than single indirection, but preserves -/// function pointer relation tests and correct behavior for function pointers -/// (all calls to 'F', direct or indirect) go the address stored in the global -/// variable at the time of the call. -/// -/// This method does *not* initialize the function implementation addresses. -/// The client must do this prior to running any call-sites that have been -/// indirected. -JITIndirections makeCallsDoubleIndirect( - llvm::Module &M, - const std::function &ShouldIndirect, - const char *JITImplSuffix, const char *JITAddrSuffix); - -/// @brief Given a set of indirections and a symbol lookup functor, create a -/// JITResolveCallbackHandler instance that will resolve the -/// implementations for the indirected symbols on demand. -template -std::unique_ptr -createCallbackHandlerFromJITIndirections(const JITIndirections &Indirs, - const PersistentMangler &NM, - SymbolLookupFtor Lookup) { - auto GetImplName = Indirs.GetImplName; - auto GetAddrName = Indirs.GetAddrName; - - std::unique_ptr J = - JITResolveCallbackHandler::create( - [=](const std::string &S) { - return Lookup(NM.getMangledName(GetImplName(S))); - }, - [=](const std::string &S, TargetAddress Addr) { - void *ImplPtr = reinterpret_cast( - Lookup(NM.getMangledName(GetAddrName(S)))); - memcpy(ImplPtr, &Addr, sizeof(TargetAddress)); - }); - - for (const auto &FuncName : Indirs.IndirectedNames) - J->addFuncName(FuncName); - - return J; -} + TargetAddress getAvailableTrampolineAddr(LLVMContext &Context) { + if (this->AvailableTrampolines.empty()) + grow(Context); + assert(!this->AvailableTrampolines.empty() && + "Failed to grow available trampolines."); + TargetAddress TrampolineAddr = this->AvailableTrampolines.back(); + this->AvailableTrampolines.pop_back(); + return TrampolineAddr; + } -/// @brief Insert callback asm into module M for the symbols managed by -/// JITResolveCallbackHandler J. -void insertX86CallbackAsm(Module &M, JITResolveCallbackHandler &J); - -/// @brief Initialize global indirects to point into the callback asm. -template -void initializeFuncAddrs(JITResolveCallbackHandler &J, - const JITIndirections &Indirs, - const PersistentMangler &NM, LookupFtor Lookup) { - // Forward declare so that we can access this, even though it's an - // implementation detail. - std::string getJITResolveCallbackIndexLabel(unsigned I); - - if (J.getNumFuncs() == 0) - return; - - // Force a look up one of the global addresses for a function that has been - // indirected. We need to do this to trigger the emission of the module - // holding the callback asm. We can't rely on that emission happening - // automatically when we look up the callback asm symbols, since lazy-emitting - // layers can't see those. - Lookup(NM.getMangledName(Indirs.GetAddrName(J.getFuncName(0)))); - - // Now update indirects to point to the JIT resolve callback asm. - for (JITResolveCallbackHandler::StubIndex I = 0; I < J.getNumFuncs(); ++I) { - TargetAddress ResolveCallbackIdxAddr = - Lookup(getJITResolveCallbackIndexLabel(I)); - void *AddrPtr = reinterpret_cast( - Lookup(NM.getMangledName(Indirs.GetAddrName(J.getFuncName(I))))); - assert(AddrPtr && "Can't find stub addr global to initialize."); - memcpy(AddrPtr, &ResolveCallbackIdxAddr, sizeof(TargetAddress)); + void grow(LLVMContext &Context) { + assert(this->AvailableTrampolines.empty() && "Growing prematurely?"); + std::unique_ptr M(new Module("trampoline_block", Context)); + auto GetLabelName = + TargetT::insertCompileCallbackTrampolines(*M, ResolverBlockAddr, + this->NumTrampolinesPerBlock, + this->ActiveTrampolines.size()); + auto H = JIT.addModuleSet(SingletonSet(std::move(M)), nullptr); + JIT.emitAndFinalize(H); + for (unsigned I = 0; I < this->NumTrampolinesPerBlock; ++I) { + std::string Name = GetLabelName(I); + auto TrampolineSymbol = JIT.findSymbolIn(H, Name, false); + assert(TrampolineSymbol && "Failed to emit trampoline."); + this->AvailableTrampolines.push_back(TrampolineSymbol.getAddress()); + } } -} -/// @brief Extract all functions matching the predicate ShouldExtract in to -/// their own modules. (Does not modify the original module.) -/// -/// @return A set of modules, the first containing all symbols (including -/// globals and aliases) that did not pass ShouldExtract, and each -/// subsequent module containing one of the functions that did meet -/// ShouldExtract. -/// -/// By adding the resulting modules separately (not as a set) to a -/// LazyEmittingLayer instance, compilation can be deferred until symbols are -/// actually needed. -std::vector> -explode(const llvm::Module &OrigMod, - const std::function &ShouldExtract); - -/// @brief Given a module that has been indirectified, break each function -/// that has been indirected out into its own module. (Does not modify -/// the original module). -/// -/// @returns A set of modules covering the symbols provided by OrigMod. -std::vector> -explode(const llvm::Module &OrigMod, const JITIndirections &Indirections); + JITLayerT &JIT; + TargetAddress ResolverBlockAddr; +}; + +GlobalVariable* createImplPointer(Function &F, const Twine &Name, + Constant *Initializer); + +void makeStub(Function &F, GlobalVariable &ImplPointer); + +typedef std::map> ModulePartitionMap; + +void partition(Module &M, const ModulePartitionMap &PMap); + +/// @brief Struct for trivial "complete" partitioning of a module. +struct FullyPartitionedModule { + std::unique_ptr GlobalVars; + std::unique_ptr Commons; + std::vector> Functions; +}; + +FullyPartitionedModule fullyPartition(Module &M); + } #endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H diff --git a/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h b/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h index cf72f3722fa..29e49c33333 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h +++ b/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h @@ -18,9 +18,37 @@ namespace llvm { -/// @brief Insert callback asm into module M for the symbols managed by -/// JITResolveCallbackHandler J. -void insertX86CallbackAsm(Module &M, JITResolveCallbackHandler &J); +class OrcX86_64 { +public: + static const char *ResolverBlockName; + + /// @brief Insert module-level inline callback asm into module M for the + /// symbols managed by JITResolveCallbackHandler J. + static void insertResolverBlock( + Module &M, + JITCompileCallbackManagerBase &JCBM); + + /// @brief Get a label name from the given index. + typedef std::function LabelNameFtor; + + static const unsigned CallSize = 6; + + /// @brief Insert the requested number of trampolines into the given module. + /// @param M Module to insert the call block into. + /// @param NumCalls Number of calls to create in the call block. + /// @param StartIndex Optional argument specifying the index suffix to start + /// with. + /// @return A functor that provides the symbol name for each entry in the call + /// block. + /// + static LabelNameFtor insertCompileCallbackTrampolines( + Module &M, + TargetAddress TrampolineAddr, + unsigned NumCalls, + unsigned StartIndex = 0); + +}; + } #endif // LLVM_EXECUTIONENGINE_ORC_ORCTARGETSUPPORT_H diff --git a/lib/ExecutionEngine/Orc/CloneSubModule.cpp b/lib/ExecutionEngine/Orc/CloneSubModule.cpp index 54acb78153e..64a33c80f20 100644 --- a/lib/ExecutionEngine/Orc/CloneSubModule.cpp +++ b/lib/ExecutionEngine/Orc/CloneSubModule.cpp @@ -27,27 +27,20 @@ void llvm::copyFunctionBody(Function &New, const Function &Orig, } } -std::unique_ptr -llvm::CloneSubModule(const Module &M, +void llvm::CloneSubModule(llvm::Module &Dst, const Module &Src, HandleGlobalVariableFtor HandleGlobalVariable, - HandleFunctionFtor HandleFunction, bool KeepInlineAsm) { + HandleFunctionFtor HandleFunction, bool CloneInlineAsm) { ValueToValueMapTy VMap; - // First off, we need to create the new module. - std::unique_ptr New = - llvm::make_unique(M.getModuleIdentifier(), M.getContext()); - - New->setDataLayout(M.getDataLayout()); - New->setTargetTriple(M.getTargetTriple()); - if (KeepInlineAsm) - New->setModuleInlineAsm(M.getModuleInlineAsm()); + if (CloneInlineAsm) + Dst.appendModuleInlineAsm(Src.getModuleInlineAsm()); // Copy global variables (but not initializers, yet). - for (Module::const_global_iterator I = M.global_begin(), E = M.global_end(); + for (Module::const_global_iterator I = Src.global_begin(), E = Src.global_end(); I != E; ++I) { GlobalVariable *GV = new GlobalVariable( - *New, I->getType()->getElementType(), I->isConstant(), I->getLinkage(), + Dst, I->getType()->getElementType(), I->isConstant(), I->getLinkage(), (Constant *)nullptr, I->getName(), (GlobalVariable *)nullptr, I->getThreadLocalMode(), I->getType()->getAddressSpace()); GV->copyAttributesFrom(I); @@ -55,21 +48,21 @@ llvm::CloneSubModule(const Module &M, } // Loop over the functions in the module, making external functions as before - for (Module::const_iterator I = M.begin(), E = M.end(); I != E; ++I) { + for (Module::const_iterator I = Src.begin(), E = Src.end(); I != E; ++I) { Function *NF = Function::Create(cast(I->getType()->getElementType()), - I->getLinkage(), I->getName(), &*New); + I->getLinkage(), I->getName(), &Dst); NF->copyAttributesFrom(I); VMap[I] = NF; } // Loop over the aliases in the module - for (Module::const_alias_iterator I = M.alias_begin(), E = M.alias_end(); + for (Module::const_alias_iterator I = Src.alias_begin(), E = Src.alias_end(); I != E; ++I) { auto *PTy = cast(I->getType()); auto *GA = GlobalAlias::create(PTy->getElementType(), PTy->getAddressSpace(), - I->getLinkage(), I->getName(), &*New); + I->getLinkage(), I->getName(), &Dst); GA->copyAttributesFrom(I); VMap[I] = GA; } @@ -77,7 +70,7 @@ llvm::CloneSubModule(const Module &M, // Now that all of the things that global variable initializer can refer to // have been created, loop through and copy the global variable referrers // over... We also set the attributes on the global now. - for (Module::const_global_iterator I = M.global_begin(), E = M.global_end(); + for (Module::const_global_iterator I = Src.global_begin(), E = Src.global_end(); I != E; ++I) { GlobalVariable &GV = *cast(VMap[I]); HandleGlobalVariable(GV, *I, VMap); @@ -85,13 +78,13 @@ llvm::CloneSubModule(const Module &M, // Similarly, copy over function bodies now... // - for (Module::const_iterator I = M.begin(), E = M.end(); I != E; ++I) { + for (Module::const_iterator I = Src.begin(), E = Src.end(); I != E; ++I) { Function &F = *cast(VMap[I]); HandleFunction(F, *I, VMap); } // And aliases - for (Module::const_alias_iterator I = M.alias_begin(), E = M.alias_end(); + for (Module::const_alias_iterator I = Src.alias_begin(), E = Src.alias_end(); I != E; ++I) { GlobalAlias *GA = cast(VMap[I]); if (const Constant *C = I->getAliasee()) @@ -99,14 +92,13 @@ llvm::CloneSubModule(const Module &M, } // And named metadata.... - for (Module::const_named_metadata_iterator I = M.named_metadata_begin(), - E = M.named_metadata_end(); + for (Module::const_named_metadata_iterator I = Src.named_metadata_begin(), + E = Src.named_metadata_end(); I != E; ++I) { const NamedMDNode &NMD = *I; - NamedMDNode *NewNMD = New->getOrInsertNamedMetadata(NMD.getName()); + NamedMDNode *NewNMD = Dst.getOrInsertNamedMetadata(NMD.getName()); for (unsigned i = 0, e = NMD.getNumOperands(); i != e; ++i) NewNMD->addOperand(MapMetadata(NMD.getOperand(i), VMap)); } - return New; } diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 2fcfb82237c..57616a54840 100644 --- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -9,149 +9,101 @@ using namespace llvm; namespace llvm { -JITIndirections makeCallsSingleIndirect( - Module &M, const std::function &ShouldIndirect, - const char *JITImplSuffix, const char *JITAddrSuffix) { - std::vector Worklist; - std::vector FuncNames; - - for (auto &F : M) - if (ShouldIndirect(F) && (F.user_begin() != F.user_end())) { - Worklist.push_back(&F); - FuncNames.push_back(F.getName()); - } - - for (auto *F : Worklist) { - GlobalVariable *FImplAddr = new GlobalVariable( - M, F->getType(), false, GlobalValue::ExternalLinkage, - Constant::getNullValue(F->getType()), F->getName() + JITAddrSuffix, - nullptr, GlobalValue::NotThreadLocal, 0, true); - FImplAddr->setVisibility(GlobalValue::HiddenVisibility); - - for (auto *U : F->users()) { - assert(isa(U) && "Cannot indirect non-instruction use"); - IRBuilder<> Builder(cast(U)); - U->replaceUsesOfWith(F, Builder.CreateLoad(FImplAddr)); - } - } - - return JITIndirections( - FuncNames, [=](StringRef S) -> std::string { return std::string(S); }, - [=](StringRef S) - -> std::string { return std::string(S) + JITAddrSuffix; }); +GlobalVariable* createImplPointer(Function &F, const Twine &Name, + Constant *Initializer) { + assert(F.getParent() && "Function isn't in a module."); + if (!Initializer) + Initializer = Constant::getNullValue(F.getType()); + Module &M = *F.getParent(); + return new GlobalVariable(M, F.getType(), false, GlobalValue::ExternalLinkage, + Initializer, Name, nullptr, + GlobalValue::NotThreadLocal, 0, true); } -JITIndirections makeCallsDoubleIndirect( - Module &M, const std::function &ShouldIndirect, - const char *JITImplSuffix, const char *JITAddrSuffix) { - - std::vector Worklist; - std::vector FuncNames; - - for (auto &F : M) - if (!F.isDeclaration() && !F.hasAvailableExternallyLinkage() && - ShouldIndirect(F)) - Worklist.push_back(&F); - - for (auto *F : Worklist) { - std::string OrigName = F->getName(); - F->setName(OrigName + JITImplSuffix); - FuncNames.push_back(OrigName); - - GlobalVariable *FImplAddr = new GlobalVariable( - M, F->getType(), false, GlobalValue::ExternalLinkage, - Constant::getNullValue(F->getType()), OrigName + JITAddrSuffix, nullptr, - GlobalValue::NotThreadLocal, 0, true); - FImplAddr->setVisibility(GlobalValue::HiddenVisibility); +void makeStub(Function &F, GlobalVariable &ImplPointer) { + assert(F.isDeclaration() && "Can't turn a definition into a stub."); + assert(F.getParent() && "Function isn't in a module."); + Module &M = *F.getParent(); + BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); + IRBuilder<> Builder(EntryBlock); + LoadInst *ImplAddr = Builder.CreateLoad(&ImplPointer); + std::vector CallArgs; + for (auto &A : F.args()) + CallArgs.push_back(&A); + CallInst *Call = Builder.CreateCall(ImplAddr, CallArgs); + Call->setTailCall(); + Builder.CreateRet(Call); +} - Function *FRedirect = - Function::Create(F->getFunctionType(), F->getLinkage(), OrigName, &M); +void partition(Module &M, const ModulePartitionMap &PMap) { - F->replaceAllUsesWith(FRedirect); + for (auto &KVPair : PMap) { - BasicBlock *EntryBlock = - BasicBlock::Create(M.getContext(), "entry", FRedirect); + auto ExtractGlobalVars = + [&](GlobalVariable &New, const GlobalVariable &Orig, + ValueToValueMapTy &VMap) { + if (KVPair.second.count(&Orig)) { + copyGVInitializer(New, Orig, VMap); + } + if (New.getLinkage() == GlobalValue::PrivateLinkage) { + New.setLinkage(GlobalValue::ExternalLinkage); + New.setVisibility(GlobalValue::HiddenVisibility); + } + }; - IRBuilder<> Builder(EntryBlock); - LoadInst *FImplLoadedAddr = Builder.CreateLoad(FImplAddr); + auto ExtractFunctions = + [&](Function &New, const Function &Orig, ValueToValueMapTy &VMap) { + if (KVPair.second.count(&Orig)) + copyFunctionBody(New, Orig, VMap); + if (New.getLinkage() == GlobalValue::InternalLinkage) { + New.setLinkage(GlobalValue::ExternalLinkage); + New.setVisibility(GlobalValue::HiddenVisibility); + } + }; - std::vector CallArgs; - for (Value &Arg : FRedirect->args()) - CallArgs.push_back(&Arg); - CallInst *Call = Builder.CreateCall(FImplLoadedAddr, CallArgs); - Call->setTailCall(); - Builder.CreateRet(Call); + CloneSubModule(*KVPair.first, M, ExtractGlobalVars, ExtractFunctions, + false); } - - return JITIndirections( - FuncNames, [=](StringRef S) - -> std::string { return std::string(S) + JITImplSuffix; }, - [=](StringRef S) - -> std::string { return std::string(S) + JITAddrSuffix; }); } -std::vector> -explode(const Module &OrigMod, - const std::function &ShouldExtract) { - - std::vector> NewModules; +FullyPartitionedModule fullyPartition(Module &M) { + FullyPartitionedModule MP; - // Split all the globals, non-indirected functions, etc. into a single module. - auto ExtractGlobalVars = [&](GlobalVariable &New, const GlobalVariable &Orig, - ValueToValueMapTy &VMap) { - copyGVInitializer(New, Orig, VMap); - if (New.getLinkage() == GlobalValue::PrivateLinkage) { - New.setLinkage(GlobalValue::ExternalLinkage); - New.setVisibility(GlobalValue::HiddenVisibility); - } - }; + ModulePartitionMap PMap; - auto ExtractNonImplFunctions = - [&](Function &New, const Function &Orig, ValueToValueMapTy &VMap) { - if (!ShouldExtract(New)) - copyFunctionBody(New, Orig, VMap); - }; + for (auto &F : M) { - NewModules.push_back(CloneSubModule(OrigMod, ExtractGlobalVars, - ExtractNonImplFunctions, true)); - - // Preserve initializers for Common linkage vars, and make private linkage - // globals external: they are now provided by the globals module extracted - // above. - auto DropGlobalVars = [&](GlobalVariable &New, const GlobalVariable &Orig, - ValueToValueMapTy &VMap) { - if (New.getLinkage() == GlobalValue::CommonLinkage) - copyGVInitializer(New, Orig, VMap); - else if (New.getLinkage() == GlobalValue::PrivateLinkage) - New.setLinkage(GlobalValue::ExternalLinkage); - }; - - // Split each 'impl' function out in to its own module. - for (const auto &Func : OrigMod) { - if (Func.isDeclaration() || !ShouldExtract(Func)) + if (F.isDeclaration()) continue; - auto ExtractNamedFunction = - [&](Function &New, const Function &Orig, ValueToValueMapTy &VMap) { - if (New.getName() == Func.getName()) - copyFunctionBody(New, Orig, VMap); - }; - - NewModules.push_back( - CloneSubModule(OrigMod, DropGlobalVars, ExtractNamedFunction, false)); + std::string NewModuleName = (M.getName() + "." + F.getName()).str(); + MP.Functions.push_back( + llvm::make_unique(NewModuleName, M.getContext())); + MP.Functions.back()->setDataLayout(M.getDataLayout()); + PMap[MP.Functions.back().get()].insert(&F); } - return NewModules; -} + MP.GlobalVars = + llvm::make_unique((M.getName() + ".globals_and_stubs").str(), + M.getContext()); + MP.GlobalVars->setDataLayout(M.getDataLayout()); -std::vector> -explode(const Module &OrigMod, const JITIndirections &Indirections) { - std::set ImplNames; + MP.Commons = + llvm::make_unique((M.getName() + ".commons").str(), M.getContext()); + MP.Commons->setDataLayout(M.getDataLayout()); - for (const auto &FuncName : Indirections.IndirectedNames) - ImplNames.insert(Indirections.GetImplName(FuncName)); + // Make sure there's at least an empty set for the stubs map or we'll fail + // to clone anything for it (including the decls). + PMap[MP.GlobalVars.get()] = ModulePartitionMap::mapped_type(); + for (auto &GV : M.globals()) + if (GV.getLinkage() == GlobalValue::CommonLinkage) + PMap[MP.Commons.get()].insert(&GV); + else + PMap[MP.GlobalVars.get()].insert(&GV); - return explode( - OrigMod, [&](const Function &F) { return ImplNames.count(F.getName()); }); + partition(M, PMap); + + return MP; } + } diff --git a/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp b/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp index 9f278f474a2..3f146459b15 100644 --- a/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp +++ b/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp @@ -1,14 +1,11 @@ #include "llvm/ADT/Triple.h" -#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" +#include "llvm/ExecutionEngine/Orc/OrcTargetSupport.h" #include using namespace llvm; namespace { -const char *JITCallbackFuncName = "call_jit_for_lazy_compile"; -const char *JITCallbackIndexLabelPrefix = "jit_resolve_"; - std::array X86GPRsToSave = {{ "rbp", "rbx", "r12", "r13", "r14", "r15", // Callee saved. "rdi", "rsi", "rdx", "rcx", "r8", "r9", // Int args. @@ -41,61 +38,90 @@ template void restoreX86Regs(OStream &OS) { OS << " popq %" << X86GPRsToSave[X86GPRsToSave.size() - i - 1] << "\n"; } -uint64_t call_jit_for_fn(JITResolveCallbackHandler *J, uint64_t FuncIdx) { - return J->resolve(FuncIdx); +template +uint64_t executeCompileCallback(JITCompileCallbackManagerBase *JCBM, + TargetAddress CallbackID) { + return JCBM->executeCompileCallback(CallbackID); } + } namespace llvm { -std::string getJITResolveCallbackIndexLabel(unsigned I) { - std::ostringstream LabelStream; - LabelStream << JITCallbackIndexLabelPrefix << I; - return LabelStream.str(); -} +const char* OrcX86_64::ResolverBlockName = "orc_resolver_block"; -void insertX86CallbackAsm(Module &M, JITResolveCallbackHandler &J) { +void OrcX86_64::insertResolverBlock( + Module &M, + JITCompileCallbackManagerBase &JCBM) { uint64_t CallbackAddr = - static_cast(reinterpret_cast(call_jit_for_fn)); + static_cast( + reinterpret_cast(executeCompileCallback)); - std::ostringstream JITCallbackAsm; + std::ostringstream AsmStream; Triple TT(M.getTargetTriple()); if (TT.getOS() == Triple::Darwin) - JITCallbackAsm << ".section __TEXT,__text,regular,pure_instructions\n" - << ".align 4, 0x90\n"; + AsmStream << ".section __TEXT,__text,regular,pure_instructions\n" + << ".align 4, 0x90\n"; else - JITCallbackAsm << ".text\n" - << ".align 16, 0x90\n"; + AsmStream << ".text\n" + << ".align 16, 0x90\n"; - JITCallbackAsm << "jit_object_addr:\n" - << " .quad " << &J << "\n" << JITCallbackFuncName << ":\n"; + AsmStream << "jit_callback_manager_addr:\n" + << " .quad " << &JCBM << "\n" + << ResolverBlockName << ":\n"; - uint64_t ReturnAddrOffset = saveX86Regs(JITCallbackAsm); + uint64_t ReturnAddrOffset = saveX86Regs(AsmStream); // Compute index, load object address, and call JIT. - JITCallbackAsm << " movq " << ReturnAddrOffset << "(%rsp), %rax\n" - << " leaq (jit_indices_start+5)(%rip), %rbx\n" - << " subq %rbx, %rax\n" - << " xorq %rdx, %rdx\n" - << " movq $5, %rbx\n" - << " divq %rbx\n" - << " movq %rax, %rsi\n" - << " leaq jit_object_addr(%rip), %rdi\n" - << " movq (%rdi), %rdi\n" - << " movabsq $" << CallbackAddr << ", %rax\n" - << " callq *%rax\n" - << " movq %rax, " << ReturnAddrOffset << "(%rsp)\n"; - - restoreX86Regs(JITCallbackAsm); - - JITCallbackAsm << " retq\n" - << "jit_indices_start:\n"; - - for (JITResolveCallbackHandler::StubIndex I = 0; I < J.getNumFuncs(); ++I) - JITCallbackAsm << getJITResolveCallbackIndexLabel(I) << ":\n" - << " callq " << JITCallbackFuncName << "\n"; - - M.appendModuleInlineAsm(JITCallbackAsm.str()); + AsmStream << " leaq jit_callback_manager_addr(%rip), %rdi\n" + << " movq (%rdi), %rdi\n" + << " movq " << ReturnAddrOffset << "(%rsp), %rsi\n" + << " movabsq $" << CallbackAddr << ", %rax\n" + << " callq *%rax\n" + << " movq %rax, " << ReturnAddrOffset << "(%rsp)\n"; + + restoreX86Regs(AsmStream); + + AsmStream << " retq\n"; + + M.appendModuleInlineAsm(AsmStream.str()); } + +OrcX86_64::LabelNameFtor +OrcX86_64::insertCompileCallbackTrampolines(Module &M, + TargetAddress ResolverBlockAddr, + unsigned NumCalls, + unsigned StartIndex) { + const char *ResolverBlockPtrName = "Lorc_resolve_block_addr"; + + std::ostringstream AsmStream; + Triple TT(M.getTargetTriple()); + + if (TT.getOS() == Triple::Darwin) + AsmStream << ".section __TEXT,__text,regular,pure_instructions\n" + << ".align 4, 0x90\n"; + else + AsmStream << ".text\n" + << ".align 16, 0x90\n"; + + AsmStream << ResolverBlockPtrName << ":\n" + << " .quad " << ResolverBlockAddr << "\n"; + + auto GetLabelName = + [=](unsigned I) { + std::ostringstream LabelStream; + LabelStream << "orc_jcc_" << (StartIndex + I); + return LabelStream.str(); + }; + + for (unsigned I = 0; I < NumCalls; ++I) + AsmStream << GetLabelName(I) << ":\n" + << " callq *" << ResolverBlockPtrName << "(%rip)\n"; + + M.appendModuleInlineAsm(AsmStream.str()); + + return GetLabelName; +} + } -- 2.34.1