X-Git-Url: http://plrg.eecs.uci.edu/git/?p=oota-llvm.git;a=blobdiff_plain;f=include%2Fllvm%2FExecutionEngine%2FOrc%2FCompileOnDemandLayer.h;h=07914e386ad213f311231a59e8b7eecfb5ed3613;hp=ff180c21b242188bae70da4b2caeb834851059e0;hb=0b6b9ceae3f1ea6b29769fa7d2a461d65ee301c5;hpb=be06dbe64749242c4b53ced14e4f0b6ed78c90d9 diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index ff180c21b24..07914e386ad 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -22,6 +22,7 @@ #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/Transforms/Utils/Cloning.h" #include +#include #include #include "llvm/Support/Debug.h" @@ -36,51 +37,86 @@ namespace orc { /// added to the layer below. When a stub is called it triggers the extraction /// of the function body from the original module. The extracted body is then /// compiled and executed. -template (Function&)>> +template class CompileOnDemandLayer { private: - // Utility class for MapValue. Only materializes declarations for global - // variables. - class GlobalDeclMaterializer : public ValueMaterializer { + template + class LambdaMaterializer final : public ValueMaterializer { public: - typedef std::set StubSet; - - GlobalDeclMaterializer(Module &Dst, const StubSet *StubsToClone = nullptr) - : Dst(Dst), StubsToClone(StubsToClone) {} - + LambdaMaterializer(MaterializerFtor M) : M(std::move(M)) {} Value* materializeValueFor(Value *V) final { - if (auto *GV = dyn_cast(V)) - return cloneGlobalVariableDecl(Dst, *GV); - else if (auto *F = dyn_cast(V)) { - auto *ClonedF = cloneFunctionDecl(Dst, *F); - if (StubsToClone && StubsToClone->count(F)) { - GlobalVariable *FnBodyPtr = - createImplPointer(*ClonedF->getType(), *ClonedF->getParent(), - ClonedF->getName() + "$orc_addr", nullptr); - makeStub(*ClonedF, *FnBodyPtr); - ClonedF->setLinkage(GlobalValue::AvailableExternallyLinkage); - ClonedF->addFnAttr(Attribute::AlwaysInline); - } - return ClonedF; - } - // Else. - return nullptr; + return M(V); } private: - Module &Dst; - const StubSet *StubsToClone; + MaterializerFtor M; }; + template + LambdaMaterializer + createLambdaMaterializer(MaterializerFtor M) { + return LambdaMaterializer(std::move(M)); + } + typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT; + class ModuleOwner { + public: + ModuleOwner() = default; + ModuleOwner(const ModuleOwner&) = delete; + ModuleOwner& operator=(const ModuleOwner&) = delete; + virtual ~ModuleOwner() { } + virtual Module& getModule() const = 0; + }; + + template + class ModuleOwnerImpl : public ModuleOwner { + public: + ModuleOwnerImpl(ModulePtrT ModulePtr) : ModulePtr(std::move(ModulePtr)) {} + Module& getModule() const override { return *ModulePtr; } + private: + ModulePtrT ModulePtr; + }; + + template + std::unique_ptr wrapOwnership(ModulePtrT ModulePtr) { + return llvm::make_unique>(std::move(ModulePtr)); + } + struct LogicalModuleResources { - std::shared_ptr SourceModule; + std::unique_ptr SourceModuleOwner; std::set StubsToClone; + std::unique_ptr StubsMgr; + + LogicalModuleResources() = default; + + // Explicit move constructor to make MSVC happy. + LogicalModuleResources(LogicalModuleResources &&Other) + : SourceModuleOwner(std::move(Other.SourceModuleOwner)), + StubsToClone(std::move(Other.StubsToClone)), + StubsMgr(std::move(Other.StubsMgr)) {} + + // Explicit move assignment to make MSVC happy. + LogicalModuleResources& operator=(LogicalModuleResources &&Other) { + SourceModuleOwner = std::move(Other.SourceModuleOwner); + StubsToClone = std::move(Other.StubsToClone); + StubsMgr = std::move(Other.StubsMgr); + } + + JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) { + if (Name.endswith("$stub_ptr") && !ExportedSymbolsOnly) { + assert(!ExportedSymbolsOnly && "Stubs are never exported"); + return StubsMgr->findPointer(Name.drop_back(9)); + } + return StubsMgr->findStub(Name, ExportedSymbolsOnly); + } + }; + + struct LogicalDylibResources { typedef std::function SymbolResolverFtor; @@ -94,15 +130,25 @@ private: typedef std::list LogicalDylibList; public: + /// @brief Handle to a set of loaded modules. typedef typename LogicalDylibList::iterator ModuleSetHandleT; + /// @brief Module partitioning functor. + typedef std::function(Function&)> PartitioningFtor; + + /// @brief Builder for IndirectStubsManagers. + typedef std::function()> + IndirectStubsManagerBuilderT; + /// @brief Construct a compile-on-demand layer instance. - CompileOnDemandLayer(BaseLayerT &BaseLayer, CompileCallbackMgrT &CallbackMgr, - PartitioningFtor Partition, + CompileOnDemandLayer(BaseLayerT &BaseLayer, PartitioningFtor Partition, + CompileCallbackMgrT &CallbackMgr, + IndirectStubsManagerBuilderT CreateIndirectStubsManager, bool CloneStubsIntoPartitions = true) - : BaseLayer(BaseLayer), CompileCallbackMgr(CallbackMgr), - Partition(Partition), + : BaseLayer(BaseLayer), Partition(Partition), + CompileCallbackMgr(CallbackMgr), + CreateIndirectStubsManager(std::move(CreateIndirectStubsManager)), CloneStubsIntoPartitions(CloneStubsIntoPartitions) {} /// @brief Add a module to the compile-on-demand layer. @@ -125,8 +171,7 @@ public: // Process each of the modules in this module set. for (auto &M : Ms) - addLogicalModule(LogicalDylibs.back(), - std::shared_ptr(std::move(M))); + addLogicalModule(LogicalDylibs.back(), std::move(M)); return std::prev(LogicalDylibs.end()); } @@ -144,6 +189,10 @@ public: /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it exists. JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) { + for (auto LDI = LogicalDylibs.begin(), LDE = LogicalDylibs.end(); + LDI != LDE; ++LDI) + if (auto Symbol = findSymbolIn(LDI, Name, ExportedSymbolsOnly)) + return Symbol; return BaseLayer.findSymbol(Name, ExportedSymbolsOnly); } @@ -156,136 +205,175 @@ public: private: - void addLogicalModule(CODLogicalDylib &LD, std::shared_ptr SrcM) { + template + void addLogicalModule(CODLogicalDylib &LD, ModulePtrT SrcMPtr) { // Bump the linkage and rename any anonymous/privote members in SrcM to // ensure that everything will resolve properly after we partition SrcM. - makeAllSymbolsExternallyAccessible(*SrcM); + makeAllSymbolsExternallyAccessible(*SrcMPtr); // Create a logical module handle for SrcM within the logical dylib. auto LMH = LD.createLogicalModule(); auto &LMResources = LD.getLogicalModuleResources(LMH); - LMResources.SourceModule = SrcM; - // Create the GVs-and-stubs module. - auto GVsAndStubsM = llvm::make_unique( - (SrcM->getName() + ".globals_and_stubs").str(), - SrcM->getContext()); - GVsAndStubsM->setDataLayout(SrcM->getDataLayout()); - ValueToValueMapTy VMap; + LMResources.SourceModuleOwner = wrapOwnership(std::move(SrcMPtr)); - // Process module and create stubs. - // We create the stubs before copying the global variables as we know the - // stubs won't refer to any globals (they only refer to their implementation - // pointer) so there's no ordering/value-mapping issues. - for (auto &F : *SrcM) { - - // Skip declarations. - if (F.isDeclaration()) - continue; - - // Record all functions defined by this module. - if (CloneStubsIntoPartitions) - LMResources.StubsToClone.insert(&F); - - // For each definition: create a callback, a stub, and a function body - // pointer. Initialize the function body pointer to point at the callback, - // and set the callback to compile the function body. - auto CCInfo = CompileCallbackMgr.getCompileCallback(SrcM->getContext()); - Function *StubF = cloneFunctionDecl(*GVsAndStubsM, F, &VMap); - GlobalVariable *FnBodyPtr = - createImplPointer(*StubF->getType(), *StubF->getParent(), - StubF->getName() + "$orc_addr", - createIRTypedAddress(*StubF->getFunctionType(), - CCInfo.getAddress())); - makeStub(*StubF, *FnBodyPtr); - CCInfo.setCompileAction( - [this, &LD, LMH, &F]() { + Module &SrcM = LMResources.SourceModuleOwner->getModule(); + + // Create the GlobalValues module. + const DataLayout &DL = SrcM.getDataLayout(); + auto GVsM = llvm::make_unique((SrcM.getName() + ".globals").str(), + SrcM.getContext()); + GVsM->setDataLayout(DL); + + // Create function stubs. + ValueToValueMapTy VMap; + { + typename IndirectStubsMgrT::StubInitsMap StubInits; + for (auto &F : SrcM) { + // Skip declarations. + if (F.isDeclaration()) + continue; + + // Record all functions defined by this module. + if (CloneStubsIntoPartitions) + LMResources.StubsToClone.insert(&F); + + // Create a callback, associate it with the stub for the function, + // and set the compile action to compile the partition containing the + // function. + auto CCInfo = CompileCallbackMgr.getCompileCallback(SrcM.getContext()); + StubInits[mangle(F.getName(), DL)] = + std::make_pair(CCInfo.getAddress(), + JITSymbolBase::flagsFromGlobalValue(F)); + CCInfo.setCompileAction([this, &LD, LMH, &F]() { return this->extractAndCompile(LD, LMH, F); }); + } + + LMResources.StubsMgr = CreateIndirectStubsManager(); + auto EC = LMResources.StubsMgr->createStubs(StubInits); + (void)EC; + // FIXME: This should be propagated back to the user. Stub creation may + // fail for remote JITs. + assert(!EC && "Error generating stubs"); } - // Now clone the global variable declarations. - GlobalDeclMaterializer GDMat(*GVsAndStubsM); - for (auto &GV : SrcM->globals()) - if (!GV.isDeclaration()) - cloneGlobalVariableDecl(*GVsAndStubsM, GV, &VMap); + // Clone global variable decls. + for (auto &GV : SrcM.globals()) + if (!GV.isDeclaration() && !VMap.count(&GV)) + cloneGlobalVariableDecl(*GVsM, GV, &VMap); // And the aliases. - for (auto &Alias : SrcM->aliases()) - cloneGlobalAlias(*GVsAndStubsM, Alias, VMap, &GDMat); + for (auto &A : SrcM.aliases()) + if (!VMap.count(&A)) + cloneGlobalAliasDecl(*GVsM, A, VMap); + + // Now we need to clone the GV and alias initializers. + + // Initializers may refer to functions declared (but not defined) in this + // module. Build a materializer to clone decls on demand. + auto Materializer = createLambdaMaterializer( + [this, &GVsM, &LMResources](Value *V) -> Value* { + if (auto *F = dyn_cast(V)) { + // Decls in the original module just get cloned. + if (F->isDeclaration()) + return cloneFunctionDecl(*GVsM, *F); + + // Definitions in the original module (which we have emitted stubs + // for at this point) get turned into a constant alias to the stub + // instead. + const DataLayout &DL = GVsM->getDataLayout(); + std::string FName = mangle(F->getName(), DL); + auto StubSym = LMResources.StubsMgr->findStub(FName, false); + unsigned PtrBitWidth = DL.getPointerTypeSizeInBits(F->getType()); + ConstantInt *StubAddr = + ConstantInt::get(GVsM->getContext(), + APInt(PtrBitWidth, StubSym.getAddress())); + Constant *Init = ConstantExpr::getCast(Instruction::IntToPtr, + StubAddr, F->getType()); + return GlobalAlias::create(F->getFunctionType(), + F->getType()->getAddressSpace(), + F->getLinkage(), F->getName(), + Init, GVsM.get()); + } + // else.... + return nullptr; + }); - // Then clone the initializers. - for (auto &GV : SrcM->globals()) + // Clone the global variable initializers. + for (auto &GV : SrcM.globals()) if (!GV.isDeclaration()) - moveGlobalVariableInitializer(GV, VMap, &GDMat); + moveGlobalVariableInitializer(GV, VMap, &Materializer); + + // Clone the global alias initializers. + for (auto &A : SrcM.aliases()) { + auto *NewA = cast(VMap[&A]); + assert(NewA && "Alias not cloned?"); + Value *Init = MapValue(A.getAliasee(), VMap, RF_None, nullptr, + &Materializer); + NewA->setAliasee(cast(Init)); + } - // Build a resolver for the stubs module and add it to the base layer. - auto GVsAndStubsResolver = createLambdaResolver( - [&LD](const std::string &Name) { + // Build a resolver for the globals module and add it to the base layer. + auto GVsResolver = createLambdaResolver( + [&LD, LMH](const std::string &Name) { + auto &LMResources = LD.getLogicalModuleResources(LMH); + if (auto Sym = LMResources.StubsMgr->findStub(Name, false)) + return RuntimeDyld::SymbolInfo(Sym.getAddress(), Sym.getFlags()); return LD.getDylibResources().ExternalSymbolResolver(Name); }, [](const std::string &Name) { return RuntimeDyld::SymbolInfo(nullptr); }); - std::vector> GVsAndStubsMSet; - GVsAndStubsMSet.push_back(std::move(GVsAndStubsM)); - auto GVsAndStubsH = - BaseLayer.addModuleSet(std::move(GVsAndStubsMSet), + std::vector> GVsMSet; + GVsMSet.push_back(std::move(GVsM)); + auto GVsH = + BaseLayer.addModuleSet(std::move(GVsMSet), llvm::make_unique(), - std::move(GVsAndStubsResolver)); - LD.addToLogicalModule(LMH, GVsAndStubsH); + std::move(GVsResolver)); + LD.addToLogicalModule(LMH, GVsH); } - static std::string Mangle(StringRef Name, const DataLayout &DL) { + static std::string mangle(StringRef Name, const DataLayout &DL) { std::string MangledName; - { - raw_string_ostream MangledNameStream(MangledName); - Mangler::getNameWithPrefix(MangledNameStream, Name, DL); - } + Mangler::getNameWithPrefix(raw_string_ostream(MangledName), Name, DL); return MangledName; } TargetAddress extractAndCompile(CODLogicalDylib &LD, LogicalModuleHandle LMH, Function &F) { - Module &SrcM = *LD.getLogicalModuleResources(LMH).SourceModule; + auto &LMResources = LD.getLogicalModuleResources(LMH); + Module &SrcM = LMResources.SourceModuleOwner->getModule(); // If F is a declaration we must already have compiled it. if (F.isDeclaration()) return 0; // Grab the name of the function being called here. - std::string CalledFnName = Mangle(F.getName(), SrcM.getDataLayout()); + std::string CalledFnName = mangle(F.getName(), SrcM.getDataLayout()); auto Part = Partition(F); auto PartH = emitPartition(LD, LMH, Part); TargetAddress CalledAddr = 0; for (auto *SubF : Part) { - std::string FName = SubF->getName(); - auto FnBodySym = - BaseLayer.findSymbolIn(PartH, Mangle(FName, SrcM.getDataLayout()), - false); - auto FnPtrSym = - BaseLayer.findSymbolIn(*LD.moduleHandlesBegin(LMH), - Mangle(FName + "$orc_addr", - SrcM.getDataLayout()), - false); + std::string FnName = mangle(SubF->getName(), SrcM.getDataLayout()); + auto FnBodySym = BaseLayer.findSymbolIn(PartH, FnName, false); assert(FnBodySym && "Couldn't find function body."); - assert(FnPtrSym && "Couldn't find function body pointer."); TargetAddress FnBodyAddr = FnBodySym.getAddress(); - void *FnPtrAddr = reinterpret_cast( - static_cast(FnPtrSym.getAddress())); // If this is the function we're calling record the address so we can // return it from this function. if (SubF == &F) CalledAddr = FnBodyAddr; - memcpy(FnPtrAddr, &FnBodyAddr, sizeof(uintptr_t)); + // Update the function body pointer for the stub. + if (auto EC = LMResources.StubsMgr->updatePointer(FnName, FnBodyAddr)) + return 0; } return CalledAddr; @@ -296,7 +384,7 @@ private: LogicalModuleHandle LMH, const PartitionT &Part) { auto &LMResources = LD.getLogicalModuleResources(LMH); - Module &SrcM = *LMResources.SourceModule; + Module &SrcM = LMResources.SourceModuleOwner->getModule(); // Create the module. std::string NewName = SrcM.getName(); @@ -308,7 +396,43 @@ private: auto M = llvm::make_unique(NewName, SrcM.getContext()); M->setDataLayout(SrcM.getDataLayout()); ValueToValueMapTy VMap; - GlobalDeclMaterializer GDM(*M, &LMResources.StubsToClone); + + auto Materializer = createLambdaMaterializer([this, &LMResources, &M, + &VMap](Value *V) -> Value * { + if (auto *GV = dyn_cast(V)) + return cloneGlobalVariableDecl(*M, *GV); + + if (auto *F = dyn_cast(V)) { + // Check whether we want to clone an available_externally definition. + if (!LMResources.StubsToClone.count(F)) + return cloneFunctionDecl(*M, *F); + + // Ok - we want an inlinable stub. For that to work we need a decl + // for the stub pointer. + auto *StubPtr = createImplPointer(*F->getType(), *M, + F->getName() + "$stub_ptr", nullptr); + auto *ClonedF = cloneFunctionDecl(*M, *F); + makeStub(*ClonedF, *StubPtr); + ClonedF->setLinkage(GlobalValue::AvailableExternallyLinkage); + ClonedF->addFnAttr(Attribute::AlwaysInline); + return ClonedF; + } + + if (auto *A = dyn_cast(V)) { + auto *Ty = A->getValueType(); + if (Ty->isFunctionTy()) + return Function::Create(cast(Ty), + GlobalValue::ExternalLinkage, A->getName(), + M.get()); + + return new GlobalVariable(*M, Ty, false, GlobalValue::ExternalLinkage, + nullptr, A->getName(), nullptr, + GlobalValue::NotThreadLocal, + A->getType()->getAddressSpace()); + } + + return nullptr; + }); // Create decls in the new module. for (auto *F : Part) @@ -316,7 +440,7 @@ private: // Move the function bodies. for (auto *F : Part) - moveFunctionBody(*F, VMap, &GDM); + moveFunctionBody(*F, VMap, &Materializer); // Create memory manager and symbol resolver. auto MemMgr = llvm::make_unique(); @@ -340,9 +464,11 @@ private: } BaseLayerT &BaseLayer; + PartitioningFtor Partition; CompileCallbackMgrT &CompileCallbackMgr; + IndirectStubsManagerBuilderT CreateIndirectStubsManager; + LogicalDylibList LogicalDylibs; - PartitioningFtor Partition; bool CloneStubsIntoPartitions; };