From ee160106a131f9b79b89c5ea3b6c0e1e1d1f019d Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Mon, 4 May 2015 22:03:10 +0000 Subject: [PATCH] [Orc] Refactor the compile-on-demand layer to make module partitioning lazy, and avoid cloning unused decls into every partition. Module partitioning showed up as a source of significant overhead when I profiled some trivial test cases. Avoiding the overhead of partitionging for uncalled functions helps to mitigate this. This change also means that it is no longer necessary to have a LazyEmittingLayer underneath the CompileOnDemand layer, since the CompileOnDemandLayer will not extract or emit function bodies until they are called. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@236465 91177308-0d34-0410-b5e6-96231b3b80d8 --- examples/Kaleidoscope/Orc/fully_lazy/toy.cpp | 22 +- .../llvm/ExecutionEngine/Orc/CloneSubModule.h | 60 -- .../Orc/CompileOnDemandLayer.h | 608 +++++++++++------- .../ExecutionEngine/Orc/IndirectionUtils.h | 145 +++-- lib/ExecutionEngine/Orc/CMakeLists.txt | 1 - lib/ExecutionEngine/Orc/CloneSubModule.cpp | 106 --- lib/ExecutionEngine/Orc/IndirectionUtils.cpp | 158 +++-- test/ExecutionEngine/OrcLazy/hello.ll | 2 +- tools/lli/OrcLazyJIT.h | 9 +- 9 files changed, 568 insertions(+), 543 deletions(-) delete mode 100644 include/llvm/ExecutionEngine/Orc/CloneSubModule.h delete mode 100644 lib/ExecutionEngine/Orc/CloneSubModule.cpp diff --git a/examples/Kaleidoscope/Orc/fully_lazy/toy.cpp b/examples/Kaleidoscope/Orc/fully_lazy/toy.cpp index b5822921b40..93de33353da 100644 --- a/examples/Kaleidoscope/Orc/fully_lazy/toy.cpp +++ b/examples/Kaleidoscope/Orc/fully_lazy/toy.cpp @@ -1214,11 +1214,11 @@ public: void removeModule(ModuleHandleT H) { LazyEmitLayer.removeModuleSet(H); } JITSymbol findSymbol(const std::string &Name) { - return LazyEmitLayer.findSymbol(Name, true); + return LazyEmitLayer.findSymbol(Name, false); } JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name) { - return LazyEmitLayer.findSymbolIn(H, Name, true); + return LazyEmitLayer.findSymbolIn(H, Name, false); } JITSymbol findUnmangledSymbol(const std::string &Name) { @@ -1276,7 +1276,7 @@ private: makeStub(*F, *FunctionBodyPointer); // Step 4) Add the module containing the stub to the JIT. - auto H = addModule(C.takeM()); + auto StubH = addModule(C.takeM()); // Step 5) Set the compile and update actions. // @@ -1289,14 +1289,20 @@ private: // The update action will update FunctionBodyPointer to point at the newly // compiled function. std::shared_ptr Fn = std::move(FnAST); - CallbackInfo.setCompileAction([this, Fn]() { + CallbackInfo.setCompileAction([this, Fn, BodyPtrName, StubH]() { auto H = addModule(IRGen(Session, *Fn)); - return findUnmangledSymbolIn(H, Fn->Proto->Name).getAddress(); + auto BodySym = findUnmangledSymbolIn(H, Fn->Proto->Name); + auto BodyPtrSym = findUnmangledSymbolIn(StubH, BodyPtrName); + assert(BodySym && "Missing function body."); + assert(BodyPtrSym && "Missing function pointer."); + auto BodyAddr = BodySym.getAddress(); + auto BodyPtr = reinterpret_cast( + static_cast(BodyPtrSym.getAddress())); + memcpy(BodyPtr, &BodyAddr, sizeof(uintptr_t)); + return BodyAddr; }); - CallbackInfo.setUpdateAction( - getLocalFPUpdater(LazyEmitLayer, H, mangle(BodyPtrName))); - return H; + return StubH; } SessionContext &Session; diff --git a/include/llvm/ExecutionEngine/Orc/CloneSubModule.h b/include/llvm/ExecutionEngine/Orc/CloneSubModule.h deleted file mode 100644 index 1bd3955e879..00000000000 --- a/include/llvm/ExecutionEngine/Orc/CloneSubModule.h +++ /dev/null @@ -1,60 +0,0 @@ -//===-- CloneSubModule.h - Utilities for extracting sub-modules -*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Contains utilities for extracting sub-modules. Useful for breaking up modules -// for lazy jitting. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_EXECUTIONENGINE_ORC_CLONESUBMODULE_H -#define LLVM_EXECUTIONENGINE_ORC_CLONESUBMODULE_H - -#include "llvm/ADT/DenseSet.h" -#include "llvm/Transforms/Utils/ValueMapper.h" -#include - -namespace llvm { - -class Function; -class GlobalVariable; -class Module; - -namespace orc { - -/// @brief Functor type for describing how CloneSubModule should mutate a -/// GlobalVariable. -typedef std::function HandleGlobalVariableFtor; - -/// @brief Functor type for describing how CloneSubModule should mutate a -/// Function. -typedef std::function - HandleFunctionFtor; - -/// @brief Copies the initializer from Orig to New. -/// -/// Type is suitable for implicit conversion to a HandleGlobalVariableFtor. -void copyGVInitializer(GlobalVariable &New, const GlobalVariable &Orig, - ValueToValueMapTy &VMap); - -/// @brief Copies the body of Orig to New. -/// -/// Type is suitable for implicit conversion to a HandleFunctionFtor. -void copyFunctionBody(Function &New, const Function &Orig, - ValueToValueMapTy &VMap); - -/// @brief Clone a subset of the module Src into Dst. -void CloneSubModule(Module &Dst, const Module &Src, - HandleGlobalVariableFtor HandleGlobalVariable, - HandleFunctionFtor HandleFunction, bool KeepInlineAsm); - -} // End namespace orc. -} // End namespace llvm. - -#endif // LLVM_EXECUTIONENGINE_ORC_CLONESUBMODULE_H diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 30f7f1cd5f5..919eba3dc5f 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -15,110 +15,180 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_COMPILEONDEMANDLAYER_H #define LLVM_EXECUTIONENGINE_ORC_COMPILEONDEMANDLAYER_H +//#include "CloneSubModule.h" #include "IndirectionUtils.h" #include "LambdaResolver.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/Transforms/Utils/Cloning.h" #include +#include + +#include "llvm/Support/Debug.h" namespace llvm { namespace orc { /// @brief Compile-on-demand layer. /// -/// Modules added to this layer have their calls indirected, and are then -/// broken up into a set of single-function modules, each of which is added -/// to the layer below in a singleton set. The lower layer can be any layer that -/// accepts IR module sets. -/// -/// It is expected that this layer will frequently be used on top of a -/// LazyEmittingLayer. The combination of the two ensures that each function is -/// compiled only when it is first called. +/// When a module is added to this layer a stub is created for each of its +/// function definitions. The stubs and other global values are immediately +/// added to the layer below. When a stub is called it triggers the extraction +/// of the function body from the original module. The extracted body is then +/// compiled and executed. template class CompileOnDemandLayer { private: - /// @brief Lookup helper that provides compatibility with the classic - /// static-compilation symbol resolution process. - /// - /// The CompileOnDemand (COD) layer splits modules up into multiple - /// sub-modules, each held in its own llvm::Module instance, in order to - /// support lazy compilation. When a module that contains private symbols is - /// broken up symbol linkage changes may be required to enable access to - /// "private" data that now resides in a different llvm::Module instance. To - /// retain expected symbol resolution behavior for clients of the COD layer, - /// the CODScopedLookup class uses a two-tiered lookup system to resolve - /// symbols. Lookup first scans sibling modules that were split from the same - /// original module (logical-module scoped lookup), then scans all other - /// modules that have been added to the lookup scope (logical-dylib scoped - /// lookup). - class CODScopedLookup { + + // Utility class for MapValue. Only materializes declarations for global + // variables. + class GlobalDeclMaterializer : public ValueMaterializer { + public: + GlobalDeclMaterializer(Module &Dst) : Dst(Dst) {} + Value* materializeValueFor(Value *V) final { + if (auto *GV = dyn_cast(V)) + return cloneGlobalVariableDecl(Dst, *GV); + else if (auto *F = dyn_cast(V)) + return cloneFunctionDecl(Dst, *F); + // Else. + return nullptr; + } private: - typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT; - typedef std::vector SiblingHandlesList; - typedef std::list PseudoDylibModuleSetHandlesList; + Module &Dst; + }; + typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT; + class UncompiledPartition; + + // Logical module. + // + // This struct contains the handles for the global values and stubs (which + // cover the external symbols of the original module), plus the handes for + // each of the extracted partitions. These handleds are used for lookup (only + // the globals/stubs module is searched) and memory management. The actual + // searching and resource management are handled by the LogicalDylib that owns + // the LogicalModule. + struct LogicalModule { + std::unique_ptr SrcM; + BaseLayerModuleSetHandleT GVsAndStubsHandle; + std::vector ImplHandles; + }; + + // Logical dylib. + // + // This class handles symbol resolution and resource management for a set of + // modules that were added together as a logical dylib. + // + // A logical dylib contains one-or-more LogicalModules plus a set of + // UncompiledPartitions. LogicalModules support symbol resolution and resource + // management for for code that has already been emitted. UncompiledPartitions + // represent code that has not yet been compiled. + class LogicalDylib { + private: + friend class UncompiledPartition; + typedef std::list LogicalModuleList; public: - /// @brief Handle for a logical module. - typedef typename PseudoDylibModuleSetHandlesList::iterator LMHandle; - /// @brief Construct a scoped lookup. - CODScopedLookup(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {} + typedef unsigned UncompiledPartitionID; + typedef typename LogicalModuleList::iterator LMHandle; + + // Construct a logical dylib. + LogicalDylib(CompileOnDemandLayer &CODLayer) : CODLayer(CODLayer) { } - virtual ~CODScopedLookup() {} + // Delete this logical dylib, release logical module resources. + virtual ~LogicalDylib() { + releaseLogicalModuleResources(); + } + + // Get a reference to the containing layer. + CompileOnDemandLayer& getCODLayer() { return CODLayer; } - /// @brief Start a new context for a single logical module. + // Get a reference to the base layer. + BaseLayerT& getBaseLayer() { return CODLayer.BaseLayer; } + + // Start a new context for a single logical module. LMHandle createLogicalModule() { - Handles.push_back(SiblingHandlesList()); - return std::prev(Handles.end()); + LogicalModules.push_back(LogicalModule()); + return std::prev(LogicalModules.end()); } - /// @brief Add a concrete Module's handle to the given logical Module's - /// lookup scope. + // Set the global-values-and-stubs module handle for this logical module. + void setGVsAndStubsHandle(LMHandle LMH, BaseLayerModuleSetHandleT H) { + LMH->GVsAndStubsHandle = H; + } + + // Return the global-values-and-stubs module handle for this logical module. + BaseLayerModuleSetHandleT getGVsAndStubsHandle(LMHandle LMH) { + return LMH->GVsAndStubsHandle; + } + + // Add a handle to a module containing lazy function bodies to the given + // logical module. void addToLogicalModule(LMHandle LMH, BaseLayerModuleSetHandleT H) { - LMH->push_back(H); + LMH->ImplHandles.push_back(H); } - /// @brief Remove a logical Module from the CODScopedLookup entirely. - void removeLogicalModule(LMHandle LMH) { Handles.erase(LMH); } + // Create an UncompiledPartition attached to this LogicalDylib. + UncompiledPartition& createUncompiledPartition(LMHandle LMH, + std::shared_ptr SrcM); + + // Take ownership of the given UncompiledPartition from the logical dylib. + std::unique_ptr + takeUPOwnership(UncompiledPartitionID ID); - /// @brief Look up a symbol in this context. - JITSymbol findSymbol(LMHandle LMH, const std::string &Name) { - if (auto Symbol = findSymbolIn(LMH, Name)) + // Look up a symbol in this context. + JITSymbol findSymbolInternally(LMHandle LMH, const std::string &Name) { + if (auto Symbol = getBaseLayer().findSymbolIn(LMH->GVsAndStubsHandle, + Name, false)) return Symbol; - for (auto I = Handles.begin(), E = Handles.end(); I != E; ++I) + for (auto I = LogicalModules.begin(), E = LogicalModules.end(); I != E; + ++I) if (I != LMH) - if (auto Symbol = findSymbolIn(I, Name)) + if (auto Symbol = getBaseLayer().findSymbolIn(I->GVsAndStubsHandle, + Name, false)) return Symbol; return nullptr; } - /// @brief Find an external symbol (via the user supplied SymbolResolver). + JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly) { + for (auto &LM : LogicalModules) + if (auto Symbol = getBaseLayer().findSymbolIn(LM.GVsAndStubsHandle, + Name, + ExportedSymbolsOnly)) + return Symbol; + return nullptr; + } + + // Find an external symbol (via the user supplied SymbolResolver). virtual RuntimeDyld::SymbolInfo - externalLookup(const std::string &Name) const = 0; + findSymbolExternally(const std::string &Name) const = 0; private: - JITSymbol findSymbolIn(LMHandle LMH, const std::string &Name) { - for (auto H : *LMH) - if (auto Symbol = BaseLayer.findSymbolIn(H, Name, false)) - return Symbol; - return nullptr; + void releaseLogicalModuleResources() { + for (auto I = LogicalModules.begin(), E = LogicalModules.end(); I != E; + ++I) { + getBaseLayer().removeModuleSet(I->GVsAndStubsHandle); + for (auto H : I->ImplHandles) + getBaseLayer().removeModuleSet(H); + } } - BaseLayerT &BaseLayer; - PseudoDylibModuleSetHandlesList Handles; + CompileOnDemandLayer &CODLayer; + LogicalModuleList LogicalModules; + std::vector> UncompiledPartitions; }; template - class CODScopedLookupImpl : public CODScopedLookup { + class LogicalDylibImpl : public LogicalDylib { public: - CODScopedLookupImpl(BaseLayerT &BaseLayer, ResolverPtrT Resolver) - : CODScopedLookup(BaseLayer), Resolver(std::move(Resolver)) {} + LogicalDylibImpl(CompileOnDemandLayer &CODLayer, ResolverPtrT Resolver) + : LogicalDylib(CODLayer), Resolver(std::move(Resolver)) {} RuntimeDyld::SymbolInfo - externalLookup(const std::string &Name) const override { + findSymbolExternally(const std::string &Name) const override { return Resolver->findSymbol(Name); } @@ -127,44 +197,169 @@ private: }; template - static std::shared_ptr - createCODScopedLookup(BaseLayerT &BaseLayer, - ResolverPtrT Resolver) { - typedef CODScopedLookupImpl Impl; - return std::make_shared(BaseLayer, std::move(Resolver)); + static std::unique_ptr + createLogicalDylib(CompileOnDemandLayer &CODLayer, + ResolverPtrT Resolver) { + typedef LogicalDylibImpl Impl; + return llvm::make_unique(CODLayer, std::move(Resolver)); } - typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT; - typedef std::vector BaseLayerModuleSetHandleListT; + // Uncompiled partition. + // + // Represents one as-yet uncompiled portion of a module. + class UncompiledPartition { + public: + + struct PartitionEntry { + PartitionEntry(Function *F, TargetAddress CallbackID) + : F(F), CallbackID(CallbackID) {} + Function *F; + TargetAddress CallbackID; + }; + + typedef std::vector PartitionEntryList; + + // Creates an uncompiled partition with the list of functions that make up + // this partition. + UncompiledPartition(LogicalDylib &LD, typename LogicalDylib::LMHandle LMH, + std::shared_ptr SrcM) + : LD(LD), LMH(LMH), SrcM(std::move(SrcM)), ID(~0U) {} + + ~UncompiledPartition() { + // FIXME: When we want to support threaded lazy compilation we'll need to + // lock the callback manager here. + auto &CCMgr = LD.getCODLayer().CompileCallbackMgr; + for (auto PEntry : PartitionEntries) + CCMgr.releaseCompileCallback(PEntry.CallbackID); + } + + // Set the ID for this partition. + void setID(typename LogicalDylib::UncompiledPartitionID ID) { + this->ID = ID; + } - struct ModuleSetInfo { - // Symbol lookup - just one for the whole module set. - std::shared_ptr Lookup; + // Set the function set and callbacks for this partition. + void setPartitionEntries(PartitionEntryList PartitionEntries) { + this->PartitionEntries = std::move(PartitionEntries); + } - // Logical module handles. - std::vector LMHandles; + // Handle a compile callback for the function at index FnIdx. + TargetAddress compile(unsigned FnIdx) { + // Take ownership of self. This will ensure we delete the partition and + // free all its resources once we're done compiling. + std::unique_ptr This = LD.takeUPOwnership(ID); + + // Release all other compile callbacks for this partition. + // We skip the callback for this function because that's the one that + // called us, and the callback manager will already have removed it. + auto &CCMgr = LD.getCODLayer().CompileCallbackMgr; + for (unsigned I = 0; I < PartitionEntries.size(); ++I) + if (I != FnIdx) + CCMgr.releaseCompileCallback(PartitionEntries[I].CallbackID); + + // Grab the name of the function being called here. + Function *F = PartitionEntries[FnIdx].F; + std::string CalledFnName = Mangle(F->getName(), SrcM->getDataLayout()); + + // Extract the function and add it to the base layer. + auto PartitionImplH = emitPartition(); + LD.addToLogicalModule(LMH, PartitionImplH); + + // Update body pointers. + // FIXME: When we start supporting remote lazy jitting this will need to + // be replaced with a user-supplied callback for updating the + // remote pointers. + TargetAddress CalledAddr = 0; + for (unsigned I = 0; I < PartitionEntries.size(); ++I) { + auto F = PartitionEntries[I].F; + std::string FName(F->getName()); + auto FnBodySym = + LD.getBaseLayer().findSymbolIn(PartitionImplH, + Mangle(FName, SrcM->getDataLayout()), + false); + auto FnPtrSym = + LD.getBaseLayer().findSymbolIn(LD.getGVsAndStubsHandle(LMH), + Mangle(FName + "$orc_addr", + SrcM->getDataLayout()), + false); + assert(FnBodySym && "Couldn't find function body."); + assert(FnPtrSym && "Couldn't find function body pointer."); + + auto FnBodyAddr = FnBodySym.getAddress(); + void *FnPtrAddr = reinterpret_cast( + static_cast(FnPtrSym.getAddress())); + + // If this is the function we're calling record the address so we can + // return it from this function. + if (I == FnIdx) + CalledAddr = FnBodyAddr; + + memcpy(FnPtrAddr, &FnBodyAddr, sizeof(uintptr_t)); + } - // List of vectors of module set handles: - // One vector per logical module - each vector holds the handles for the - // exploded modules for that logical module in the base layer. - BaseLayerModuleSetHandleListT BaseLayerModuleSetHandles; + // Finally, clear the partition structure so we don't try to + // double-release the callbacks in the UncompiledPartition destructor. + PartitionEntries.clear(); - ModuleSetInfo(std::shared_ptr Lookup) - : Lookup(std::move(Lookup)) {} + return CalledAddr; + } - void releaseResources(BaseLayerT &BaseLayer) { - for (auto LMH : LMHandles) - Lookup->removeLogicalModule(LMH); - for (auto H : BaseLayerModuleSetHandles) - BaseLayer.removeModuleSet(H); + private: + + BaseLayerModuleSetHandleT emitPartition() { + // Create the module. + std::string NewName(SrcM->getName()); + for (auto &PEntry : PartitionEntries) { + NewName += "."; + NewName += PEntry.F->getName(); + } + auto PM = llvm::make_unique(NewName, SrcM->getContext()); + PM->setDataLayout(SrcM->getDataLayout()); + ValueToValueMapTy VMap; + GlobalDeclMaterializer GDM(*PM); + + // Create decls in the new module. + for (auto &PEntry : PartitionEntries) + cloneFunctionDecl(*PM, *PEntry.F, &VMap); + + // Move the function bodies. + for (auto &PEntry : PartitionEntries) + moveFunctionBody(*PEntry.F, VMap); + + // Create memory manager and symbol resolver. + auto MemMgr = llvm::make_unique(); + auto Resolver = createLambdaResolver( + [this](const std::string &Name) { + if (auto Symbol = LD.findSymbolInternally(LMH, Name)) + return RuntimeDyld::SymbolInfo(Symbol.getAddress(), + Symbol.getFlags()); + return LD.findSymbolExternally(Name); + }, + [this](const std::string &Name) { + if (auto Symbol = LD.findSymbolInternally(LMH, Name)) + return RuntimeDyld::SymbolInfo(Symbol.getAddress(), + Symbol.getFlags()); + return RuntimeDyld::SymbolInfo(nullptr); + }); + std::vector> PartMSet; + PartMSet.push_back(std::move(PM)); + return LD.getBaseLayer().addModuleSet(std::move(PartMSet), + std::move(MemMgr), + std::move(Resolver)); } + + LogicalDylib &LD; + typename LogicalDylib::LMHandle LMH; + std::shared_ptr SrcM; + typename LogicalDylib::UncompiledPartitionID ID; + PartitionEntryList PartitionEntries; }; - typedef std::list ModuleSetInfoListT; + typedef std::list> LogicalDylibList; public: /// @brief Handle to a set of loaded modules. - typedef typename ModuleSetInfoListT::iterator ModuleSetHandleT; + typedef typename LogicalDylibList::iterator ModuleSetHandleT; /// @brief Construct a compile-on-demand layer instance. CompileOnDemandLayer(BaseLayerT &BaseLayer, CompileCallbackMgrT &CallbackMgr) @@ -180,19 +375,23 @@ public: assert(MemMgr == nullptr && "User supplied memory managers not supported with COD yet."); - // Create a lookup context and ModuleSetInfo for this module set. - // For the purposes of symbol resolution the set Ms will be treated as if - // the modules it contained had been linked together as a dylib. - auto DylibLookup = createCODScopedLookup(BaseLayer, std::move(Resolver)); - ModuleSetHandleT H = - ModuleSetInfos.insert(ModuleSetInfos.end(), ModuleSetInfo(DylibLookup)); - ModuleSetInfo &MSI = ModuleSetInfos.back(); + LogicalDylibs.push_back(createLogicalDylib(*this, std::move(Resolver))); // Process each of the modules in this module set. - for (auto &M : Ms) - partitionAndAdd(*M, MSI); + for (auto &M : Ms) { + std::vector> Partitioning; + for (auto &F : *M) { + if (F.isDeclaration()) + continue; + Partitioning.push_back(std::vector()); + Partitioning.back().push_back(&F); + } + addLogicalModule(*LogicalDylibs.back(), + std::shared_ptr(std::move(M)), + std::move(Partitioning)); + } - return H; + return std::prev(LogicalDylibs.end()); } /// @brief Remove the module represented by the given handle. @@ -200,8 +399,7 @@ public: /// This will remove all modules in the layers below that were derived from /// the module represented by H. void removeModuleSet(ModuleSetHandleT H) { - H->releaseResources(BaseLayer); - ModuleSetInfos.erase(H); + LogicalDylibs.erase(H); } /// @brief Search for the given named symbol. @@ -216,149 +414,85 @@ public: /// below this one. JITSymbol findSymbolIn(ModuleSetHandleT H, const std::string &Name, bool ExportedSymbolsOnly) { - - for (auto &BH : H->BaseLayerModuleSetHandles) { - if (auto Symbol = BaseLayer.findSymbolIn(BH, Name, ExportedSymbolsOnly)) - return Symbol; - } - return nullptr; + return (*H)->findSymbol(Name, ExportedSymbolsOnly); } private: - void partitionAndAdd(Module &M, ModuleSetInfo &MSI) { - const char *AddrSuffix = "$orc_addr"; - const char *BodySuffix = "$orc_body"; - - // We're going to break M up into a bunch of sub-modules, but we want - // internal linkage symbols to still resolve sensibly. CODScopedLookup - // provides the "logical module" concept to make this work, so create a - // new logical module for M. - auto DylibLookup = MSI.Lookup; - auto LogicalModule = DylibLookup->createLogicalModule(); - MSI.LMHandles.push_back(LogicalModule); - - // Partition M into a "globals and stubs" module, a "common symbols" module, - // and a list of single-function modules. - auto PartitionedModule = fullyPartition(M); - auto StubsModule = std::move(PartitionedModule.GlobalVars); - auto CommonsModule = std::move(PartitionedModule.Commons); - auto FunctionModules = std::move(PartitionedModule.Functions); - - // Emit the commons stright away. - auto CommonHandle = addModule(std::move(CommonsModule), MSI, LogicalModule); - BaseLayer.emitAndFinalize(CommonHandle); - - // Map of definition names to callback-info data structures. We'll use - // this to build the compile actions for the stubs below. - typedef std::map - StubInfoMap; - StubInfoMap StubInfos; - - // Now we need to take each of the extracted Modules and add them to - // base layer. Each Module will be added individually to make sure they - // can be compiled separately, and each will get its own lookaside - // memory manager that will resolve within this logical module first. - for (auto &SubM : FunctionModules) { - - // Keep track of the stubs we create for this module so that we can set - // their compile actions. - std::vector NewStubInfos; - - // Search for function definitions and insert stubs into the stubs - // module. - for (auto &F : *SubM) { - if (F.isDeclaration()) - continue; - - std::string Name = F.getName(); - Function *Proto = StubsModule->getFunction(Name); - assert(Proto && "Failed to clone function decl into stubs module."); - auto CallbackInfo = - CompileCallbackMgr.getCompileCallback(Proto->getContext()); - GlobalVariable *FunctionBodyPointer = - createImplPointer(*Proto->getType(), *Proto->getParent(), - Name + AddrSuffix, - createIRTypedAddress(*Proto->getFunctionType(), - CallbackInfo.getAddress())); - makeStub(*Proto, *FunctionBodyPointer); - - F.setName(Name + BodySuffix); - F.setVisibility(GlobalValue::HiddenVisibility); - - auto KV = std::make_pair(std::move(Name), std::move(CallbackInfo)); - NewStubInfos.push_back(StubInfos.insert(StubInfos.begin(), KV)); - } - - auto H = addModule(std::move(SubM), MSI, LogicalModule); - - // Set the compile actions for this module: - for (auto &KVPair : NewStubInfos) { - std::string BodyName = Mangle(KVPair->first + BodySuffix, - M.getDataLayout()); - auto &CCInfo = KVPair->second; - CCInfo.setCompileAction( - [=](){ - return BaseLayer.findSymbolIn(H, BodyName, false).getAddress(); - }); + void addLogicalModule(LogicalDylib &LD, std::shared_ptr SrcM, + std::vector> Partitions) { + + // Bump the linkage and rename any anonymous/privote members in SrcM to + // ensure that everything will resolve properly after we partition SrcM. + makeAllSymbolsExternallyAccessible(*SrcM); + + // Create a logical module handle for SrcM within the logical dylib. + auto LMH = LD.createLogicalModule(); + + // Create the GVs-and-stubs module. + auto GVsAndStubsM = llvm::make_unique( + (SrcM->getName() + ".globals_and_stubs").str(), + SrcM->getContext()); + GVsAndStubsM->setDataLayout(SrcM->getDataLayout()); + ValueToValueMapTy VMap; + + // Process partitions and create stubs. + // We create the stubs before copying the global variables as we know the + // stubs won't refer to any globals (they only refer to their implementation + // pointer) so there's no ordering/value-mapping issues. + for (auto& Partition : Partitions) { + auto &UP = LD.createUncompiledPartition(LMH, SrcM); + typename UncompiledPartition::PartitionEntryList PartitionEntries; + for (auto &F : Partition) { + assert(!F->isDeclaration() && + "Partition should only contain definitions"); + unsigned FnIdx = PartitionEntries.size(); + auto CCI = CompileCallbackMgr.getCompileCallback(SrcM->getContext()); + PartitionEntries.push_back( + typename UncompiledPartition::PartitionEntry(F, CCI.getAddress())); + Function *StubF = cloneFunctionDecl(*GVsAndStubsM, *F, &VMap); + GlobalVariable *FnBodyPtr = + createImplPointer(*StubF->getType(), *StubF->getParent(), + StubF->getName() + "$orc_addr", + createIRTypedAddress(*StubF->getFunctionType(), + CCI.getAddress())); + makeStub(*StubF, *FnBodyPtr); + CCI.setCompileAction([&UP, FnIdx]() { return UP.compile(FnIdx); }); } + UP.setPartitionEntries(std::move(PartitionEntries)); } - // Ok - we've processed all the partitioned modules. Now add the - // stubs/globals module and set the update actions. - auto StubsH = - addModule(std::move(StubsModule), MSI, LogicalModule); - - for (auto &KVPair : StubInfos) { - std::string AddrName = Mangle(KVPair.first + AddrSuffix, - M.getDataLayout()); - auto &CCInfo = KVPair.second; - CCInfo.setUpdateAction( - getLocalFPUpdater(BaseLayer, StubsH, AddrName)); - } - } - - // Add the given Module to the base layer using a memory manager that will - // perform the appropriate scoped lookup (i.e. will look first with in the - // module from which it was extracted, then into the set to which that module - // belonged, and finally externally). - BaseLayerModuleSetHandleT addModule( - std::unique_ptr M, - ModuleSetInfo &MSI, - typename CODScopedLookup::LMHandle LogicalModule) { - - // Add this module to the JIT with a memory manager that uses the - // DylibLookup to resolve symbols. - std::vector> MSet; - MSet.push_back(std::move(M)); - - auto DylibLookup = MSI.Lookup; - auto Resolver = - createLambdaResolver( - [=](const std::string &Name) { - if (auto Symbol = DylibLookup->findSymbol(LogicalModule, Name)) + // Now clone the global variable declarations. + GlobalDeclMaterializer GDMat(*GVsAndStubsM); + for (auto &GV : SrcM->globals()) + if (!GV.isDeclaration()) + cloneGlobalVariableDecl(*GVsAndStubsM, GV, &VMap); + + // Then clone the initializers. + for (auto &GV : SrcM->globals()) + if (!GV.isDeclaration()) + moveGlobalVariableInitializer(GV, VMap, &GDMat); + + // Build a resolver for the stubs module and add it to the base layer. + auto GVsAndStubsResolver = createLambdaResolver( + [&LD](const std::string &Name) { + if (auto Symbol = LD.findSymbol(Name, false)) return RuntimeDyld::SymbolInfo(Symbol.getAddress(), Symbol.getFlags()); - return DylibLookup->externalLookup(Name); + return LD.findSymbolExternally(Name); }, - [=](const std::string &Name) -> RuntimeDyld::SymbolInfo { - if (auto Symbol = DylibLookup->findSymbol(LogicalModule, Name)) - return RuntimeDyld::SymbolInfo(Symbol.getAddress(), - Symbol.getFlags()); - return nullptr; + [&LD](const std::string &Name) { + return RuntimeDyld::SymbolInfo(nullptr); }); - BaseLayerModuleSetHandleT H = - BaseLayer.addModuleSet(std::move(MSet), - make_unique(), - std::move(Resolver)); - // Add this module to the logical module lookup. - DylibLookup->addToLogicalModule(LogicalModule, H); - MSI.BaseLayerModuleSetHandles.push_back(H); - - return H; + std::vector> GVsAndStubsMSet; + GVsAndStubsMSet.push_back(std::move(GVsAndStubsM)); + auto GVsAndStubsH = + BaseLayer.addModuleSet(std::move(GVsAndStubsMSet), + llvm::make_unique(), + std::move(GVsAndStubsResolver)); + LD.setGVsAndStubsHandle(LMH, GVsAndStubsH); } static std::string Mangle(StringRef Name, const DataLayout &DL) { @@ -373,9 +507,33 @@ private: BaseLayerT &BaseLayer; CompileCallbackMgrT &CompileCallbackMgr; - ModuleSetInfoListT ModuleSetInfos; + LogicalDylibList LogicalDylibs; }; +template +typename CompileOnDemandLayer:: + UncompiledPartition& +CompileOnDemandLayer::LogicalDylib:: + createUncompiledPartition(LMHandle LMH, std::shared_ptr SrcM) { + UncompiledPartitions.push_back( + llvm::make_unique(*this, LMH, std::move(SrcM))); + UncompiledPartitions.back()->setID(UncompiledPartitions.size() - 1); + return *UncompiledPartitions.back(); +} + +template +std::unique_ptr:: + UncompiledPartition> +CompileOnDemandLayer::LogicalDylib:: + takeUPOwnership(UncompiledPartitionID ID) { + + std::swap(UncompiledPartitions[ID], UncompiledPartitions.back()); + UncompiledPartitions[ID]->setID(ID); + auto UP = std::move(UncompiledPartitions.back()); + UncompiledPartitions.pop_back(); + return UP; +} + } // End namespace orc. } // End namespace llvm. diff --git a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index 13ba125e80b..4b7fc5e84b9 100644 --- a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -21,6 +21,7 @@ #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" +#include "llvm/Transforms/Utils/ValueMapper.h" #include namespace llvm { @@ -32,28 +33,22 @@ class JITCompileCallbackManagerBase { public: typedef std::function CompileFtor; - typedef std::function UpdateFtor; /// @brief Handle to a newly created compile callback. Can be used to get an /// IR constant representing the address of the trampoline, and to set - /// the compile and update actions for the callback. + /// the compile action for the callback. class CompileCallbackInfo { public: - CompileCallbackInfo(TargetAddress Addr, CompileFtor &Compile, - UpdateFtor &Update) - : Addr(Addr), Compile(Compile), Update(Update) {} + CompileCallbackInfo(TargetAddress Addr, CompileFtor &Compile) + : Addr(Addr), Compile(Compile) {} TargetAddress getAddress() const { return Addr; } void setCompileAction(CompileFtor Compile) { this->Compile = std::move(Compile); } - void setUpdateAction(UpdateFtor Update) { - this->Update = std::move(Update); - } private: TargetAddress Addr; CompileFtor &Compile; - UpdateFtor &Update; }; /// @brief Construct a JITCompileCallbackManagerBase. @@ -71,8 +66,8 @@ public: /// @brief Execute the callback for the given trampoline id. Called by the JIT /// to compile functions on demand. - TargetAddress executeCompileCallback(TargetAddress TrampolineID) { - TrampolineMapT::iterator I = ActiveTrampolines.find(TrampolineID); + TargetAddress executeCompileCallback(TargetAddress TrampolineAddr) { + auto I = ActiveTrampolines.find(TrampolineAddr); // FIXME: Also raise an error in the Orc error-handler when we finally have // one. if (I == ActiveTrampolines.end()) @@ -84,31 +79,43 @@ public: // Moving the trampoline ID back to the available list first means there's at // least one available trampoline if the compile action triggers a request for // a new one. - AvailableTrampolines.push_back(I->first); - auto CallbackHandler = std::move(I->second); + auto Compile = std::move(I->second); ActiveTrampolines.erase(I); + AvailableTrampolines.push_back(TrampolineAddr); - if (auto Addr = CallbackHandler.Compile()) { - CallbackHandler.Update(Addr); + if (auto Addr = Compile()) return Addr; - } + return ErrorHandlerAddress; } - /// @brief Get/create a compile callback with the given signature. + /// @brief Reserve a compile callback. virtual CompileCallbackInfo getCompileCallback(LLVMContext &Context) = 0; -protected: + /// @brief Get a CompileCallbackInfo for an existing callback. + CompileCallbackInfo getCompileCallbackInfo(TargetAddress TrampolineAddr) { + auto I = ActiveTrampolines.find(TrampolineAddr); + assert(I != ActiveTrampolines.end() && "Not an active trampoline."); + return CompileCallbackInfo(I->first, I->second); + } - struct CallbackHandler { - CompileFtor Compile; - UpdateFtor Update; - }; + /// @brief Release a compile callback. + /// + /// Note: Callbacks are auto-released after they execute. This method should + /// only be called to manually release a callback that is not going to + /// execute. + void releaseCompileCallback(TargetAddress TrampolineAddr) { + auto I = ActiveTrampolines.find(TrampolineAddr); + assert(I != ActiveTrampolines.end() && "Not an active trampoline."); + ActiveTrampolines.erase(I); + AvailableTrampolines.push_back(TrampolineAddr); + } +protected: TargetAddress ErrorHandlerAddress; unsigned NumTrampolinesPerBlock; - typedef std::map TrampolineMapT; + typedef std::map TrampolineMapT; TrampolineMapT ActiveTrampolines; std::vector AvailableTrampolines; }; @@ -140,11 +147,8 @@ public: /// @brief Get/create a compile callback with the given signature. CompileCallbackInfo getCompileCallback(LLVMContext &Context) final { TargetAddress TrampolineAddr = getAvailableTrampolineAddr(Context); - auto &CallbackHandler = - this->ActiveTrampolines[TrampolineAddr]; - - return CompileCallbackInfo(TrampolineAddr, CallbackHandler.Compile, - CallbackHandler.Update); + auto &Compile = this->ActiveTrampolines[TrampolineAddr]; + return CompileCallbackInfo(TrampolineAddr, Compile); } private: @@ -218,22 +222,6 @@ private: TargetAddress ResolverBlockAddr; }; -/// @brief Get an update functor that updates the value of a named function -/// pointer. -template -JITCompileCallbackManagerBase::UpdateFtor -getLocalFPUpdater(JITLayerT &JIT, typename JITLayerT::ModuleSetHandleT H, - std::string Name) { - // FIXME: Move-capture Name once we can use C++14. - return [=,&JIT](TargetAddress Addr) { - auto FPSym = JIT.findSymbolIn(H, Name, true); - assert(FPSym && "Cannot find function pointer to update."); - void *FPAddr = reinterpret_cast( - static_cast(FPSym.getAddress())); - memcpy(FPAddr, &Addr, sizeof(uintptr_t)); - }; - } - /// @brief Build a function pointer of FunctionType with the given constant /// address. /// @@ -250,27 +238,56 @@ GlobalVariable* createImplPointer(PointerType &PT, Module &M, /// indirect call using the given function pointer. void makeStub(Function &F, GlobalVariable &ImplPointer); -typedef std::map> ModulePartitionMap; - -/// @brief Extract subsections of a Module into the given Module according to -/// the given ModulePartitionMap. -void partition(Module &M, const ModulePartitionMap &PMap); - -/// @brief Struct for trivial "complete" partitioning of a module. -class FullyPartitionedModule { -public: - std::unique_ptr GlobalVars; - std::unique_ptr Commons; - std::vector> Functions; - - FullyPartitionedModule() = default; - FullyPartitionedModule(FullyPartitionedModule &&S) - : GlobalVars(std::move(S.GlobalVars)), Commons(std::move(S.Commons)), - Functions(std::move(S.Functions)) {} -}; +/// @brief Raise linkage types and rename as necessary to ensure that all +/// symbols are accessible for other modules. +/// +/// This should be called before partitioning a module to ensure that the +/// partitions retain access to each other's symbols. +void makeAllSymbolsExternallyAccessible(Module &M); -/// @brief Extract every function in M into a separate module. -FullyPartitionedModule fullyPartition(Module &M); +/// @brief Clone a function declaration into a new module. +/// +/// This function can be used as the first step towards creating a callback +/// stub (see makeStub), or moving a function body (see moveFunctionBody). +/// +/// If the VMap argument is non-null, a mapping will be added between F and +/// the new declaration, and between each of F's arguments and the new +/// declaration's arguments. This map can then be passed in to moveFunction to +/// move the function body if required. Note: When moving functions between +/// modules with these utilities, all decls should be cloned (and added to a +/// single VMap) before any bodies are moved. This will ensure that references +/// between functions all refer to the versions in the new module. +Function* cloneFunctionDecl(Module &Dst, const Function &F, + ValueToValueMapTy *VMap = nullptr); + +/// @brief Move the body of function 'F' to a cloned function declaration in a +/// different module (See related cloneFunctionDecl). +/// +/// If the target function declaration is not supplied via the NewF parameter +/// then it will be looked up via the VMap. +/// +/// This will delete the body of function 'F' from its original parent module, +/// but leave its declaration. +void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, + ValueMaterializer *Materializer = nullptr, + Function *NewF = nullptr); + +/// @brief Clone a global variable declaration into a new module. +GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, + ValueToValueMapTy *VMap = nullptr); + +/// @brief Move global variable GV from its parent module to cloned global +/// declaration in a different module. +/// +/// If the target global declaration is not supplied via the NewGV parameter +/// then it will be looked up via the VMap. +/// +/// This will delete the initializer of GV from its original parent module, +/// but leave its declaration. +void moveGlobalVariableInitializer(GlobalVariable &OrigGV, + ValueToValueMapTy &VMap, + ValueMaterializer *Materializer = nullptr, + GlobalVariable *NewGV = nullptr); } // End namespace orc. } // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/CMakeLists.txt b/lib/ExecutionEngine/Orc/CMakeLists.txt index b38b4594057..18f0441c466 100644 --- a/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -1,5 +1,4 @@ add_llvm_library(LLVMOrcJIT - CloneSubModule.cpp ExecutionUtils.cpp IndirectionUtils.cpp OrcMCJITReplacement.cpp diff --git a/lib/ExecutionEngine/Orc/CloneSubModule.cpp b/lib/ExecutionEngine/Orc/CloneSubModule.cpp deleted file mode 100644 index c9810097d7d..00000000000 --- a/lib/ExecutionEngine/Orc/CloneSubModule.cpp +++ /dev/null @@ -1,106 +0,0 @@ -#include "llvm/ExecutionEngine/Orc/CloneSubModule.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/GlobalVariable.h" -#include "llvm/IR/Module.h" -#include "llvm/Transforms/Utils/Cloning.h" - -namespace llvm { -namespace orc { - -void copyGVInitializer(GlobalVariable &New, const GlobalVariable &Orig, - ValueToValueMapTy &VMap) { - if (Orig.hasInitializer()) - New.setInitializer(MapValue(Orig.getInitializer(), VMap)); -} - -void copyFunctionBody(Function &New, const Function &Orig, - ValueToValueMapTy &VMap) { - if (!Orig.isDeclaration()) { - Function::arg_iterator DestI = New.arg_begin(); - for (Function::const_arg_iterator J = Orig.arg_begin(); J != Orig.arg_end(); - ++J) { - DestI->setName(J->getName()); - VMap[J] = DestI++; - } - - SmallVector Returns; // Ignore returns cloned. - CloneFunctionInto(&New, &Orig, VMap, /*ModuleLevelChanges=*/true, Returns); - } -} - -void CloneSubModule(llvm::Module &Dst, const Module &Src, - HandleGlobalVariableFtor HandleGlobalVariable, - HandleFunctionFtor HandleFunction, bool CloneInlineAsm) { - - ValueToValueMapTy VMap; - - if (CloneInlineAsm) - Dst.appendModuleInlineAsm(Src.getModuleInlineAsm()); - - // Copy global variables (but not initializers, yet). - for (Module::const_global_iterator I = Src.global_begin(), E = Src.global_end(); - I != E; ++I) { - GlobalVariable *GV = new GlobalVariable( - Dst, I->getType()->getElementType(), I->isConstant(), I->getLinkage(), - (Constant *)nullptr, I->getName(), (GlobalVariable *)nullptr, - I->getThreadLocalMode(), I->getType()->getAddressSpace()); - GV->copyAttributesFrom(I); - VMap[I] = GV; - } - - // Loop over the functions in the module, making external functions as before - for (Module::const_iterator I = Src.begin(), E = Src.end(); I != E; ++I) { - Function *NF = - Function::Create(cast(I->getType()->getElementType()), - I->getLinkage(), I->getName(), &Dst); - NF->copyAttributesFrom(I); - VMap[I] = NF; - } - - // Loop over the aliases in the module - for (Module::const_alias_iterator I = Src.alias_begin(), E = Src.alias_end(); - I != E; ++I) { - auto *PTy = cast(I->getType()); - auto *GA = GlobalAlias::create(PTy, I->getLinkage(), I->getName(), &Dst); - GA->copyAttributesFrom(I); - VMap[I] = GA; - } - - // Now that all of the things that global variable initializer can refer to - // have been created, loop through and copy the global variable referrers - // over... We also set the attributes on the global now. - for (Module::const_global_iterator I = Src.global_begin(), E = Src.global_end(); - I != E; ++I) { - GlobalVariable &GV = *cast(VMap[I]); - HandleGlobalVariable(GV, *I, VMap); - } - - // Similarly, copy over function bodies now... - // - for (Module::const_iterator I = Src.begin(), E = Src.end(); I != E; ++I) { - Function &F = *cast(VMap[I]); - HandleFunction(F, *I, VMap); - } - - // And aliases - for (Module::const_alias_iterator I = Src.alias_begin(), E = Src.alias_end(); - I != E; ++I) { - GlobalAlias *GA = cast(VMap[I]); - if (const Constant *C = I->getAliasee()) - GA->setAliasee(MapValue(C, VMap)); - } - - // And named metadata.... - for (Module::const_named_metadata_iterator I = Src.named_metadata_begin(), - E = Src.named_metadata_end(); - I != E; ++I) { - const NamedMDNode &NMD = *I; - NamedMDNode *NewNMD = Dst.getOrInsertNamedMetadata(NMD.getName()); - for (unsigned i = 0, e = NMD.getNumOperands(); i != e; ++i) - NewNMD->addOperand(MapMetadata(NMD.getOperand(i), VMap)); - } - -} - -} // End namespace orc. -} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 75b610876b7..4ed87303148 100644 --- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -9,10 +9,10 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" -#include "llvm/ExecutionEngine/Orc/CloneSubModule.h" #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/Transforms/Utils/Cloning.h" #include #include @@ -32,9 +32,11 @@ GlobalVariable* createImplPointer(PointerType &PT, Module &M, const Twine &Name, Constant *Initializer) { if (!Initializer) Initializer = Constant::getNullValue(&PT); - return new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, - Initializer, Name, nullptr, - GlobalValue::NotThreadLocal, 0, true); + auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, + Initializer, Name, nullptr, + GlobalValue::NotThreadLocal, 0, true); + IP->setVisibility(GlobalValue::HiddenVisibility); + return IP; } void makeStub(Function &F, GlobalVariable &ImplPointer) { @@ -50,7 +52,10 @@ void makeStub(Function &F, GlobalVariable &ImplPointer) { CallInst *Call = Builder.CreateCall(ImplAddr, CallArgs); Call->setTailCall(); Call->setAttributes(F.getAttributes()); - Builder.CreateRet(Call); + if (F.getReturnType()->isVoidTy()) + Builder.CreateRetVoid(); + else + Builder.CreateRet(Call); } // Utility class for renaming global values and functions during partitioning. @@ -84,83 +89,94 @@ private: DenseMap Names; }; -void partition(Module &M, const ModulePartitionMap &PMap) { - - GlobalRenamer Renamer; - - for (auto &KVPair : PMap) { - - auto ExtractGlobalVars = - [&](GlobalVariable &New, const GlobalVariable &Orig, - ValueToValueMapTy &VMap) { - if (KVPair.second.count(&Orig)) { - copyGVInitializer(New, Orig, VMap); - } - if (New.hasLocalLinkage()) { - if (Renamer.needsRenaming(New)) - New.setName(Renamer.getRename(Orig)); - New.setLinkage(GlobalValue::ExternalLinkage); - New.setVisibility(GlobalValue::HiddenVisibility); - } - assert(!Renamer.needsRenaming(New) && "Invalid global name."); - }; - - auto ExtractFunctions = - [&](Function &New, const Function &Orig, ValueToValueMapTy &VMap) { - if (KVPair.second.count(&Orig)) - copyFunctionBody(New, Orig, VMap); - if (New.hasLocalLinkage()) { - if (Renamer.needsRenaming(New)) - New.setName(Renamer.getRename(Orig)); - New.setLinkage(GlobalValue::ExternalLinkage); - New.setVisibility(GlobalValue::HiddenVisibility); - } - assert(!Renamer.needsRenaming(New) && "Invalid function name."); - }; - - CloneSubModule(*KVPair.first, M, ExtractGlobalVars, ExtractFunctions, - false); +static void raiseVisibilityOnValue(GlobalValue &V, GlobalRenamer &R) { + if (V.hasLocalLinkage()) { + if (R.needsRenaming(V)) + V.setName(R.getRename(V)); + V.setLinkage(GlobalValue::ExternalLinkage); + V.setVisibility(GlobalValue::HiddenVisibility); } + V.setUnnamedAddr(false); + assert(!R.needsRenaming(V) && "Invalid global name."); } -FullyPartitionedModule fullyPartition(Module &M) { - FullyPartitionedModule MP; - - ModulePartitionMap PMap; +void makeAllSymbolsExternallyAccessible(Module &M) { + GlobalRenamer Renamer; - for (auto &F : M) { + for (auto &F : M) + raiseVisibilityOnValue(F, Renamer); - if (F.isDeclaration()) - continue; + for (auto &GV : M.globals()) + raiseVisibilityOnValue(GV, Renamer); +} - std::string NewModuleName = (M.getName() + "." + F.getName()).str(); - MP.Functions.push_back( - llvm::make_unique(NewModuleName, M.getContext())); - MP.Functions.back()->setDataLayout(M.getDataLayout()); - PMap[MP.Functions.back().get()].insert(&F); +Function* cloneFunctionDecl(Module &Dst, const Function &F, + ValueToValueMapTy *VMap) { + assert(F.getParent() != &Dst && "Can't copy decl over existing function."); + Function *NewF = + Function::Create(cast(F.getType()->getElementType()), + F.getLinkage(), F.getName(), &Dst); + NewF->copyAttributesFrom(&F); + + if (VMap) { + (*VMap)[&F] = NewF; + auto NewArgI = NewF->arg_begin(); + for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; + ++ArgI, ++NewArgI) + (*VMap)[ArgI] = NewArgI; } - MP.GlobalVars = - llvm::make_unique((M.getName() + ".globals_and_stubs").str(), - M.getContext()); - MP.GlobalVars->setDataLayout(M.getDataLayout()); - - MP.Commons = - llvm::make_unique((M.getName() + ".commons").str(), M.getContext()); - MP.Commons->setDataLayout(M.getDataLayout()); + return NewF; +} - // Make sure there's at least an empty set for the stubs map or we'll fail - // to clone anything for it (including the decls). - PMap[MP.GlobalVars.get()] = ModulePartitionMap::mapped_type(); - for (auto &GV : M.globals()) - if (GV.getLinkage() == GlobalValue::CommonLinkage) - PMap[MP.Commons.get()].insert(&GV); - else - PMap[MP.GlobalVars.get()].insert(&GV); +void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, + ValueMaterializer *Materializer, + Function *NewF) { + assert(!OrigF.isDeclaration() && "Nothing to move"); + if (!NewF) + NewF = cast(VMap[&OrigF]); + else + assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap."); + assert(NewF && "Function mapping missing from VMap."); + assert(NewF->getParent() != OrigF.getParent() && + "moveFunctionBody should only be used to move bodies between " + "modules."); + + SmallVector Returns; // Ignore returns cloned. + CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns, + "", nullptr, nullptr, Materializer); + OrigF.deleteBody(); +} - partition(M, PMap); +GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, + ValueToValueMapTy *VMap) { + assert(GV.getParent() != &Dst && "Can't copy decl over existing global var."); + GlobalVariable *NewGV = new GlobalVariable( + Dst, GV.getType()->getElementType(), GV.isConstant(), + GV.getLinkage(), nullptr, GV.getName(), nullptr, + GV.getThreadLocalMode(), GV.getType()->getAddressSpace()); + NewGV->copyAttributesFrom(&GV); + if (VMap) + (*VMap)[&GV] = NewGV; + return NewGV; +} - return MP; +void moveGlobalVariableInitializer(GlobalVariable &OrigGV, + ValueToValueMapTy &VMap, + ValueMaterializer *Materializer, + GlobalVariable *NewGV) { + assert(OrigGV.hasInitializer() && "Nothing to move"); + if (!NewGV) + NewGV = cast(VMap[&OrigGV]); + else + assert(VMap[&OrigGV] == NewGV && + "Incorrect global variable mapping in VMap."); + assert(NewGV->getParent() != OrigGV.getParent() && + "moveGlobalVariable should only be used to move initializers between " + "modules"); + + NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None, + nullptr, Materializer)); } } // End namespace orc. diff --git a/test/ExecutionEngine/OrcLazy/hello.ll b/test/ExecutionEngine/OrcLazy/hello.ll index 795224e019b..c3cf0f346a1 100644 --- a/test/ExecutionEngine/OrcLazy/hello.ll +++ b/test/ExecutionEngine/OrcLazy/hello.ll @@ -1,7 +1,7 @@ ; RUN: lli -jit-kind=orc-lazy -orc-lazy-debug=funcs-to-stdout %s | FileCheck %s ; ; CHECK: Hello -; CHECK: [ {{.*}}main$orc_body ] +; CHECK: [ {{.*}}main ] ; CHECK: Goodbye %class.Foo = type { i8 } diff --git a/tools/lli/OrcLazyJIT.h b/tools/lli/OrcLazyJIT.h index 2b2db6e668c..bff2eca26a5 100644 --- a/tools/lli/OrcLazyJIT.h +++ b/tools/lli/OrcLazyJIT.h @@ -21,7 +21,6 @@ #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" -#include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" #include "llvm/IR/LLVMContext.h" @@ -37,9 +36,7 @@ public: typedef std::function(std::unique_ptr)> TransformFtor; typedef orc::IRTransformLayer IRDumpLayerT; - typedef orc::LazyEmittingLayer LazyEmitLayerT; - typedef orc::CompileOnDemandLayer CODLayerT; + typedef orc::CompileOnDemandLayer CODLayerT; typedef CODLayerT::ModuleSetHandleT ModuleHandleT; typedef std::function< @@ -57,9 +54,8 @@ public: ObjectLayer(), CompileLayer(ObjectLayer, orc::SimpleCompiler(*this->TM)), IRDumpLayer(CompileLayer, createDebugDumper()), - LazyEmitLayer(IRDumpLayer), CCMgr(BuildCallbackMgr(IRDumpLayer, CCMgrMemMgr, Context)), - CODLayer(LazyEmitLayer, *CCMgr), + CODLayer(IRDumpLayer, *CCMgr), CXXRuntimeOverrides([this](const std::string &S) { return mangle(S); }) {} ~OrcLazyJIT() { @@ -154,7 +150,6 @@ private: ObjLayerT ObjectLayer; CompileLayerT CompileLayer; IRDumpLayerT IRDumpLayer; - LazyEmitLayerT LazyEmitLayer; std::unique_ptr CCMgr; CODLayerT CODLayer; -- 2.34.1