X-Git-Url: http://plrg.eecs.uci.edu/git/?p=oota-llvm.git;a=blobdiff_plain;f=lib%2FExecutionEngine%2FJIT%2FJITEmitter.cpp;h=ba5d4e42addc7314fbe832c4ce9962a3a9690278;hp=89131a0dde7f216a39fe20fb91bb8b80cf493f3d;hb=eb3d76da81e2148ed7c577594c873ba147f4f435;hpb=a3f99f90338d89354384ca25f53ca4450a1a9d18 diff --git a/lib/ExecutionEngine/JIT/JITEmitter.cpp b/lib/ExecutionEngine/JIT/JITEmitter.cpp index 89131a0dde7..ba5d4e42add 100644 --- a/lib/ExecutionEngine/JIT/JITEmitter.cpp +++ b/lib/ExecutionEngine/JIT/JITEmitter.cpp @@ -14,32 +14,39 @@ #define DEBUG_TYPE "jit" #include "JIT.h" -#include "JITDwarfEmitter.h" -#include "llvm/Constants.h" -#include "llvm/Module.h" -#include "llvm/DerivedTypes.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/OwningPtr.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/ValueMap.h" #include "llvm/CodeGen/JITCodeEmitter.h" -#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineCodeInfo.h" #include "llvm/CodeGen/MachineConstantPool.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRelocation.h" -#include "llvm/ExecutionEngine/JITMemoryManager.h" +#include "llvm/DebugInfo.h" #include "llvm/ExecutionEngine/GenericValue.h" -#include "llvm/CodeGen/MachineCodeInfo.h" -#include "llvm/Target/TargetData.h" -#include "llvm/Target/TargetJITInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" +#include "llvm/ExecutionEngine/JITEventListener.h" +#include "llvm/ExecutionEngine/JITMemoryManager.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/ValueHandle.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Disassembler.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Memory.h" #include "llvm/Support/MutexGuard.h" -#include "llvm/Support/ValueHandle.h" -#include "llvm/System/Disassembler.h" -#include "llvm/System/Memory.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" +#include "llvm/Target/TargetJITInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" #include #ifndef NDEBUG #include @@ -48,53 +55,120 @@ using namespace llvm; STATISTIC(NumBytes, "Number of bytes of machine code compiled"); STATISTIC(NumRelos, "Number of relocations applied"); -static JIT *TheJIT = 0; +STATISTIC(NumRetries, "Number of retries with more memory"); + +// A declaration may stop being a declaration once it's fully read from bitcode. +// This function returns true if F is fully read and is still a declaration. +static bool isNonGhostDeclaration(const Function *F) { + return F->isDeclaration() && !F->isMaterializable(); +} //===----------------------------------------------------------------------===// // JIT lazy compilation code. // namespace { + class JITEmitter; + class JITResolverState; + + template + struct NoRAUWValueMapConfig : public ValueMapConfig { + typedef JITResolverState *ExtraData; + static void onRAUW(JITResolverState *, Value *Old, Value *New) { + llvm_unreachable("The JIT doesn't know how to handle a" + " RAUW on a value it has emitted."); + } + }; + + struct CallSiteValueMapConfig : public NoRAUWValueMapConfig { + typedef JITResolverState *ExtraData; + static void onDelete(JITResolverState *JRS, Function *F); + }; + class JITResolverState { public: - typedef std::map, void*> FunctionToStubMapTy; - typedef std::map StubToFunctionMapTy; + typedef ValueMap > + FunctionToLazyStubMapTy; + typedef std::map > CallSiteToFunctionMapTy; + typedef ValueMap, + CallSiteValueMapConfig> FunctionToCallSitesMapTy; typedef std::map, void*> GlobalToIndirectSymMapTy; private: - /// FunctionToStubMap - Keep track of the stub created for a particular - /// function so that we can reuse them if necessary. - FunctionToStubMapTy FunctionToStubMap; + /// FunctionToLazyStubMap - Keep track of the lazy stub created for a + /// particular function so that we can reuse them if necessary. + FunctionToLazyStubMapTy FunctionToLazyStubMap; - /// StubToFunctionMap - Keep track of the function that each stub - /// corresponds to. - StubToFunctionMapTy StubToFunctionMap; + /// CallSiteToFunctionMap - Keep track of the function that each lazy call + /// site corresponds to, and vice versa. + CallSiteToFunctionMapTy CallSiteToFunctionMap; + FunctionToCallSitesMapTy FunctionToCallSitesMap; /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a /// particular GlobalVariable so that we can reuse them if necessary. GlobalToIndirectSymMapTy GlobalToIndirectSymMap; +#ifndef NDEBUG + /// Instance of the JIT this ResolverState serves. + JIT *TheJIT; +#endif + public: - FunctionToStubMapTy& getFunctionToStubMap(const MutexGuard& locked) { + JITResolverState(JIT *jit) : FunctionToLazyStubMap(this), + FunctionToCallSitesMap(this) { +#ifndef NDEBUG + TheJIT = jit; +#endif + } + + FunctionToLazyStubMapTy& getFunctionToLazyStubMap( + const MutexGuard& locked) { assert(locked.holds(TheJIT->lock)); - return FunctionToStubMap; + return FunctionToLazyStubMap; + } + + GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap(const MutexGuard& lck) { + assert(lck.holds(TheJIT->lock)); + return GlobalToIndirectSymMap; } - StubToFunctionMapTy& getStubToFunctionMap(const MutexGuard& locked) { + std::pair LookupFunctionFromCallSite( + const MutexGuard &locked, void *CallSite) const { assert(locked.holds(TheJIT->lock)); - return StubToFunctionMap; + + // The address given to us for the stub may not be exactly right, it + // might be a little bit after the stub. As such, use upper_bound to + // find it. + CallSiteToFunctionMapTy::const_iterator I = + CallSiteToFunctionMap.upper_bound(CallSite); + assert(I != CallSiteToFunctionMap.begin() && + "This is not a known call site!"); + --I; + return *I; } - GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap(const MutexGuard& locked) { + void AddCallSite(const MutexGuard &locked, void *CallSite, Function *F) { assert(locked.holds(TheJIT->lock)); - return GlobalToIndirectSymMap; + + bool Inserted = CallSiteToFunctionMap.insert( + std::make_pair(CallSite, F)).second; + (void)Inserted; + assert(Inserted && "Pair was already in CallSiteToFunctionMap"); + FunctionToCallSitesMap[F].insert(CallSite); } + + void EraseAllCallSitesForPrelocked(Function *F); + + // Erases _all_ call sites regardless of their function. This is used to + // unregister the stub addresses from the StubToResolverMap in + // ~JITResolver(). + void EraseAllCallSitesPrelocked(); }; /// JITResolver - Keep track of, and resolve, call sites for functions that /// have not yet been compiled. class JITResolver { - typedef JITResolverState::FunctionToStubMapTy FunctionToStubMapTy; - typedef JITResolverState::StubToFunctionMapTy StubToFunctionMapTy; + typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy; + typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy; typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy; /// LazyResolverFn - The target lazy resolver function that we actually @@ -103,36 +177,37 @@ namespace { JITResolverState state; - /// ExternalFnToStubMap - This is the equivalent of FunctionToStubMap for - /// external functions. + /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap + /// for external functions. TODO: Of course, external functions don't need + /// a lazy stub. It's actually here to make it more likely that far calls + /// succeed, but no single stub can guarantee that. I'll remove this in a + /// subsequent checkin when I actually fix far calls. std::map ExternalFnToStubMap; /// revGOTMap - map addresses to indexes in the GOT std::map revGOTMap; unsigned nextGOTIndex; - static JITResolver *TheJITResolver; - public: - explicit JITResolver(JIT &jit) : nextGOTIndex(0) { - TheJIT = &jit; + JITEmitter &JE; + + /// Instance of JIT corresponding to this Resolver. + JIT *TheJIT; + public: + explicit JITResolver(JIT &jit, JITEmitter &je) + : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) { LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); - assert(TheJITResolver == 0 && "Multiple JIT resolvers?"); - TheJITResolver = this; - } - - ~JITResolver() { - TheJITResolver = 0; } - /// getFunctionStubIfAvailable - This returns a pointer to a function stub - /// if it has already been created. - void *getFunctionStubIfAvailable(Function *F); + ~JITResolver(); + + /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's + /// lazy-compilation stub if it has already been created. + void *getLazyFunctionStubIfAvailable(Function *F); - /// getFunctionStub - This returns a pointer to a function stub, creating - /// one on demand as needed. If empty is true, create a function stub - /// pointing at address 0, to be filled in later. - void *getFunctionStub(Function *F); + /// getLazyFunctionStub - This returns a pointer to a function's + /// lazy-compilation stub, creating one on demand as needed. + void *getLazyFunctionStub(Function *F); /// getExternalFunctionStub - Return a stub for the function at the /// specified address, created lazily on demand. @@ -142,21 +217,6 @@ namespace { /// specified GV address. void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress); - /// AddCallbackAtLocation - If the target is capable of rewriting an - /// instruction without the use of a stub, record the location of the use so - /// we know which function is being used at the location. - void *AddCallbackAtLocation(Function *F, void *Location) { - MutexGuard locked(TheJIT->lock); - /// Get the target-specific JIT resolver function. - state.getStubToFunctionMap(locked)[Location] = F; - return (void*)(intptr_t)LazyResolverFn; - } - - void getRelocatableGVs(SmallVectorImpl &GVs, - SmallVectorImpl &Ptrs); - - GlobalValue *invalidateStub(void *Stub); - /// getGOTIndexForAddress - Return a new or existing index in the GOT for /// an address. This function only manages slots, it does not manage the /// contents of the slots or the memory associated with the GOT. @@ -167,50 +227,314 @@ namespace { /// been compiled, this function compiles it first. static void *JITCompilerFn(void *Stub); }; + + class StubToResolverMapTy { + /// Map a stub address to a specific instance of a JITResolver so that + /// lazily-compiled functions can find the right resolver to use. + /// + /// Guarded by Lock. + std::map Map; + + /// Guards Map from concurrent accesses. + mutable sys::Mutex Lock; + + public: + /// Registers a Stub to be resolved by Resolver. + void RegisterStubResolver(void *Stub, JITResolver *Resolver) { + MutexGuard guard(Lock); + Map.insert(std::make_pair(Stub, Resolver)); + } + /// Unregisters the Stub when it's invalidated. + void UnregisterStubResolver(void *Stub) { + MutexGuard guard(Lock); + Map.erase(Stub); + } + /// Returns the JITResolver instance that owns the Stub. + JITResolver *getResolverFromStub(void *Stub) const { + MutexGuard guard(Lock); + // The address given to us for the stub may not be exactly right, it might + // be a little bit after the stub. As such, use upper_bound to find it. + // This is the same trick as in LookupFunctionFromCallSite from + // JITResolverState. + std::map::const_iterator I = Map.upper_bound(Stub); + assert(I != Map.begin() && "This is not a known stub!"); + --I; + return I->second; + } + /// True if any stubs refer to the given resolver. Only used in an assert(). + /// O(N) + bool ResolverHasStubs(JITResolver* Resolver) const { + MutexGuard guard(Lock); + for (std::map::const_iterator I = Map.begin(), + E = Map.end(); I != E; ++I) { + if (I->second == Resolver) + return true; + } + return false; + } + }; + /// This needs to be static so that a lazy call stub can access it with no + /// context except the address of the stub. + ManagedStatic StubToResolverMap; + + /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is + /// used to output functions to memory for execution. + class JITEmitter : public JITCodeEmitter { + JITMemoryManager *MemMgr; + + // When outputting a function stub in the context of some other function, we + // save BufferBegin/BufferEnd/CurBufferPtr here. + uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; + + // When reattempting to JIT a function after running out of space, we store + // the estimated size of the function we're trying to JIT here, so we can + // ask the memory manager for at least this much space. When we + // successfully emit the function, we reset this back to zero. + uintptr_t SizeEstimate; + + /// Relocations - These are the relocations that the function needs, as + /// emitted. + std::vector Relocations; + + /// MBBLocations - This vector is a mapping from MBB ID's to their address. + /// It is filled in by the StartMachineBasicBlock callback and queried by + /// the getMachineBasicBlockAddress callback. + std::vector MBBLocations; + + /// ConstantPool - The constant pool for the current function. + /// + MachineConstantPool *ConstantPool; + + /// ConstantPoolBase - A pointer to the first entry in the constant pool. + /// + void *ConstantPoolBase; + + /// ConstPoolAddresses - Addresses of individual constant pool entries. + /// + SmallVector ConstPoolAddresses; + + /// JumpTable - The jump tables for the current function. + /// + MachineJumpTableInfo *JumpTable; + + /// JumpTableBase - A pointer to the first entry in the jump table. + /// + void *JumpTableBase; + + /// Resolver - This contains info about the currently resolved functions. + JITResolver Resolver; + + /// LabelLocations - This vector is a mapping from Label ID's to their + /// address. + DenseMap LabelLocations; + + /// MMI - Machine module info for exception informations + MachineModuleInfo* MMI; + + // CurFn - The llvm function being emitted. Only valid during + // finishFunction(). + const Function *CurFn; + + /// Information about emitted code, which is passed to the + /// JITEventListeners. This is reset in startFunction and used in + /// finishFunction. + JITEvent_EmittedFunctionDetails EmissionDetails; + + struct EmittedCode { + void *FunctionBody; // Beginning of the function's allocation. + void *Code; // The address the function's code actually starts at. + void *ExceptionTable; + EmittedCode() : FunctionBody(0), Code(0), ExceptionTable(0) {} + }; + struct EmittedFunctionConfig : public ValueMapConfig { + typedef JITEmitter *ExtraData; + static void onDelete(JITEmitter *, const Function*); + static void onRAUW(JITEmitter *, const Function*, const Function*); + }; + ValueMap EmittedFunctions; + + DebugLoc PrevDL; + + /// Instance of the JIT + JIT *TheJIT; + + public: + JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM) + : SizeEstimate(0), Resolver(jit, *this), MMI(0), CurFn(0), + EmittedFunctions(this), TheJIT(&jit) { + MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); + if (jit.getJITInfo().needsGOT()) { + MemMgr->AllocateGOT(); + DEBUG(dbgs() << "JIT is managing a GOT\n"); + } + + } + ~JITEmitter() { + delete MemMgr; + } + + JITResolver &getJITResolver() { return Resolver; } + + virtual void startFunction(MachineFunction &F); + virtual bool finishFunction(MachineFunction &F); + + void emitConstantPool(MachineConstantPool *MCP); + void initJumpTableInfo(MachineJumpTableInfo *MJTI); + void emitJumpTableInfo(MachineJumpTableInfo *MJTI); + + void startGVStub(const GlobalValue* GV, + unsigned StubSize, unsigned Alignment = 1); + void startGVStub(void *Buffer, unsigned StubSize); + void finishGVStub(); + virtual void *allocIndirectGV(const GlobalValue *GV, + const uint8_t *Buffer, size_t Size, + unsigned Alignment); + + /// allocateSpace - Reserves space in the current block if any, or + /// allocate a new one of the given size. + virtual void *allocateSpace(uintptr_t Size, unsigned Alignment); + + /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, + /// this method does not allocate memory in the current output buffer, + /// because a global may live longer than the current function. + virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment); + + virtual void addRelocation(const MachineRelocation &MR) { + Relocations.push_back(MR); + } + + virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) { + if (MBBLocations.size() <= (unsigned)MBB->getNumber()) + MBBLocations.resize((MBB->getNumber()+1)*2); + MBBLocations[MBB->getNumber()] = getCurrentPCValue(); + if (MBB->hasAddressTaken()) + TheJIT->addPointerToBasicBlock(MBB->getBasicBlock(), + (void*)getCurrentPCValue()); + DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at [" + << (void*) getCurrentPCValue() << "]\n"); + } + + virtual uintptr_t getConstantPoolEntryAddress(unsigned Entry) const; + virtual uintptr_t getJumpTableEntryAddress(unsigned Entry) const; + + virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const{ + assert(MBBLocations.size() > (unsigned)MBB->getNumber() && + MBBLocations[MBB->getNumber()] && "MBB not emitted!"); + return MBBLocations[MBB->getNumber()]; + } + + /// retryWithMoreMemory - Log a retry and deallocate all memory for the + /// given function. Increase the minimum allocation size so that we get + /// more memory next time. + void retryWithMoreMemory(MachineFunction &F); + + /// deallocateMemForFunction - Deallocate all memory for the specified + /// function body. + void deallocateMemForFunction(const Function *F); + + virtual void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn); + + virtual void emitLabel(MCSymbol *Label) { + LabelLocations[Label] = getCurrentPCValue(); + } + + virtual DenseMap *getLabelLocations() { + return &LabelLocations; + } + + virtual uintptr_t getLabelAddress(MCSymbol *Label) const { + assert(LabelLocations.count(Label) && "Label not emitted!"); + return LabelLocations.find(Label)->second; + } + + virtual void setModuleInfo(MachineModuleInfo* Info) { + MMI = Info; + } + + private: + void *getPointerToGlobal(GlobalValue *GV, void *Reference, + bool MayNeedFarStub); + void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference); + }; +} + +void CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) { + JRS->EraseAllCallSitesForPrelocked(F); +} + +void JITResolverState::EraseAllCallSitesForPrelocked(Function *F) { + FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F); + if (F2C == FunctionToCallSitesMap.end()) + return; + StubToResolverMapTy &S2RMap = *StubToResolverMap; + for (SmallPtrSet::const_iterator I = F2C->second.begin(), + E = F2C->second.end(); I != E; ++I) { + S2RMap.UnregisterStubResolver(*I); + bool Erased = CallSiteToFunctionMap.erase(*I); + (void)Erased; + assert(Erased && "Missing call site->function mapping"); + } + FunctionToCallSitesMap.erase(F2C); } -JITResolver *JITResolver::TheJITResolver = 0; +void JITResolverState::EraseAllCallSitesPrelocked() { + StubToResolverMapTy &S2RMap = *StubToResolverMap; + for (CallSiteToFunctionMapTy::const_iterator + I = CallSiteToFunctionMap.begin(), + E = CallSiteToFunctionMap.end(); I != E; ++I) { + S2RMap.UnregisterStubResolver(I->first); + } + CallSiteToFunctionMap.clear(); + FunctionToCallSitesMap.clear(); +} + +JITResolver::~JITResolver() { + // No need to lock because we're in the destructor, and state isn't shared. + state.EraseAllCallSitesPrelocked(); + assert(!StubToResolverMap->ResolverHasStubs(this) && + "Resolver destroyed with stubs still alive."); +} -/// getFunctionStubIfAvailable - This returns a pointer to a function stub +/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub /// if it has already been created. -void *JITResolver::getFunctionStubIfAvailable(Function *F) { +void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) { MutexGuard locked(TheJIT->lock); // If we already have a stub for this function, recycle it. - void *&Stub = state.getFunctionToStubMap(locked)[F]; - return Stub; + return state.getFunctionToLazyStubMap(locked).lookup(F); } /// getFunctionStub - This returns a pointer to a function stub, creating /// one on demand as needed. -void *JITResolver::getFunctionStub(Function *F) { +void *JITResolver::getLazyFunctionStub(Function *F) { MutexGuard locked(TheJIT->lock); - // If we already have a stub for this function, recycle it. - void *&Stub = state.getFunctionToStubMap(locked)[F]; + // If we already have a lazy stub for this function, recycle it. + void *&Stub = state.getFunctionToLazyStubMap(locked)[F]; if (Stub) return Stub; - // Call the lazy resolver function unless we are JIT'ing non-lazily, in which - // case we must resolve the symbol now. - void *Actual = TheJIT->isLazyCompilationDisabled() - ? (void *)0 : (void *)(intptr_t)LazyResolverFn; - + // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we + // must resolve the symbol now. + void *Actual = TheJIT->isCompilingLazily() + ? (void *)(intptr_t)LazyResolverFn : (void *)0; + // If this is an external declaration, attempt to resolve the address now // to place in the stub. - if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode()) { + if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) { Actual = TheJIT->getPointerToFunction(F); // If we resolved the symbol to a null address (eg. a weak external) - // don't emit a stub. Return a null pointer to the application. If dlsym - // stubs are enabled, not being able to resolve the address is not - // meaningful. - if (!Actual && !TheJIT->areDlsymStubsEnabled()) return 0; + // don't emit a stub. Return a null pointer to the application. + if (!Actual) return 0; } + TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); + JE.startGVStub(F, SL.Size, SL.Alignment); // Codegen a new stub, calling the lazy resolver or the actual address of the // external function, if it was resolved. - Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, - *TheJIT->getCodeEmitter()); + Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE); + JE.finishGVStub(); if (Actual != (void*)(intptr_t)LazyResolverFn) { // If we are getting the stub for an external function, we really want the @@ -219,20 +543,26 @@ void *JITResolver::getFunctionStub(Function *F) { TheJIT->updateGlobalMapping(F, Stub); } - DOUT << "JIT: Stub emitted at [" << Stub << "] for function '" - << F->getName() << "'\n"; - - // Finally, keep track of the stub-to-Function mapping so that the - // JITCompilerFn knows which function to compile! - state.getStubToFunctionMap(locked)[Stub] = F; - - // If we are JIT'ing non-lazily but need to call a function that does not - // exist yet, add it to the JIT's work list so that we can fill in the stub - // address later. - if (!Actual && TheJIT->isLazyCompilationDisabled()) - if (!F->isDeclaration() || F->hasNotBeenReadFromBitcode()) - TheJIT->addPendingFunction(F); - + DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '" + << F->getName() << "'\n"); + + if (TheJIT->isCompilingLazily()) { + // Register this JITResolver as the one corresponding to this call site so + // JITCompilerFn will be able to find it. + StubToResolverMap->RegisterStubResolver(Stub, this); + + // Finally, keep track of the stub-to-Function mapping so that the + // JITCompilerFn knows which function to compile! + state.AddCallSite(locked, Stub, F); + } else if (!Actual) { + // If we are JIT'ing non-lazily but need to call a function that does not + // exist yet, add it to the JIT's work list so that we can fill in the + // stub address later. + assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() && + "'Actual' should have been set above."); + TheJIT->addPendingFunction(F); + } + return Stub; } @@ -247,10 +577,10 @@ void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) { // Otherwise, codegen a new indirect symbol. IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress, - *TheJIT->getCodeEmitter()); + JE); - DOUT << "JIT: Indirect symbol emitted at [" << IndirectSym << "] for GV '" - << GV->getName() << "'\n"; + DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym + << "] for GV '" << GV->getName() << "'\n"); return IndirectSym; } @@ -262,11 +592,13 @@ void *JITResolver::getExternalFunctionStub(void *FnAddr) { void *&Stub = ExternalFnToStubMap[FnAddr]; if (Stub) return Stub; - Stub = TheJIT->getJITInfo().emitFunctionStub(0, FnAddr, - *TheJIT->getCodeEmitter()); + TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); + JE.startGVStub(0, SL.Size, SL.Alignment); + Stub = TheJIT->getJITInfo().emitFunctionStub(0, FnAddr, JE); + JE.finishGVStub(); - DOUT << "JIT: Stub emitted at [" << Stub - << "] for external function at '" << FnAddr << "'\n"; + DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub + << "] for external function at '" << FnAddr << "'\n"); return Stub; } @@ -275,76 +607,19 @@ unsigned JITResolver::getGOTIndexForAddr(void* addr) { if (!idx) { idx = ++nextGOTIndex; revGOTMap[addr] = idx; - DOUT << "JIT: Adding GOT entry " << idx << " for addr [" << addr << "]\n"; + DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr [" + << addr << "]\n"); } return idx; } -void JITResolver::getRelocatableGVs(SmallVectorImpl &GVs, - SmallVectorImpl &Ptrs) { - MutexGuard locked(TheJIT->lock); - - FunctionToStubMapTy &FM = state.getFunctionToStubMap(locked); - GlobalToIndirectSymMapTy &GM = state.getGlobalToIndirectSymMap(locked); - - for (FunctionToStubMapTy::iterator i = FM.begin(), e = FM.end(); i != e; ++i){ - Function *F = i->first; - if (F->isDeclaration() && F->hasExternalLinkage()) { - GVs.push_back(i->first); - Ptrs.push_back(i->second); - } - } - for (GlobalToIndirectSymMapTy::iterator i = GM.begin(), e = GM.end(); - i != e; ++i) { - GVs.push_back(i->first); - Ptrs.push_back(i->second); - } -} - -GlobalValue *JITResolver::invalidateStub(void *Stub) { - MutexGuard locked(TheJIT->lock); - - FunctionToStubMapTy &FM = state.getFunctionToStubMap(locked); - StubToFunctionMapTy &SM = state.getStubToFunctionMap(locked); - GlobalToIndirectSymMapTy &GM = state.getGlobalToIndirectSymMap(locked); - - // Look up the cheap way first, to see if it's a function stub we are - // invalidating. If so, remove it from both the forward and reverse maps. - if (SM.find(Stub) != SM.end()) { - Function *F = SM[Stub]; - SM.erase(Stub); - FM.erase(F); - return F; - } - - // Otherwise, it might be an indirect symbol stub. Find it and remove it. - for (GlobalToIndirectSymMapTy::iterator i = GM.begin(), e = GM.end(); - i != e; ++i) { - if (i->second != Stub) - continue; - GlobalValue *GV = i->first; - GM.erase(i); - return GV; - } - - // Lastly, check to see if it's in the ExternalFnToStubMap. - for (std::map::iterator i = ExternalFnToStubMap.begin(), - e = ExternalFnToStubMap.end(); i != e; ++i) { - if (i->second != Stub) - continue; - ExternalFnToStubMap.erase(i); - break; - } - - return 0; -} - /// JITCompilerFn - This function is called when a lazy compilation stub has /// been entered. It looks up which function this stub corresponds to, compiles /// it if necessary, then returns the resultant function pointer. void *JITResolver::JITCompilerFn(void *Stub) { - JITResolver &JR = *TheJITResolver; - + JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub); + assert(JR && "Unable to find the corresponding JITResolver to the call site"); + Function* F = 0; void* ActualPtr = 0; @@ -352,50 +627,45 @@ void *JITResolver::JITCompilerFn(void *Stub) { // Only lock for getting the Function. The call getPointerToFunction made // in this function might trigger function materializing, which requires // JIT lock to be unlocked. - MutexGuard locked(TheJIT->lock); - - // The address given to us for the stub may not be exactly right, it might be - // a little bit after the stub. As such, use upper_bound to find it. - StubToFunctionMapTy::iterator I = - JR.state.getStubToFunctionMap(locked).upper_bound(Stub); - assert(I != JR.state.getStubToFunctionMap(locked).begin() && - "This is not a known stub!"); - F = (--I)->second; - ActualPtr = I->first; + MutexGuard locked(JR->TheJIT->lock); + + // The address given to us for the stub may not be exactly right, it might + // be a little bit after the stub. As such, use upper_bound to find it. + std::pair I = + JR->state.LookupFunctionFromCallSite(locked, Stub); + F = I.second; + ActualPtr = I.first; } // If we have already code generated the function, just return the address. - void *Result = TheJIT->getPointerToGlobalIfAvailable(F); - + void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F); + if (!Result) { // Otherwise we don't have it, do lazy compilation now. - + // If lazy compilation is disabled, emit a useful error message and abort. - if (TheJIT->isLazyCompilationDisabled()) { - cerr << "LLVM JIT requested to do lazy compilation of function '" - << F->getName() << "' when lazy compiles are disabled!\n"; - abort(); + if (!JR->TheJIT->isCompilingLazily()) { + report_fatal_error("LLVM JIT requested to do lazy compilation of" + " function '" + + F->getName() + "' when lazy compiles are disabled!"); } - - // We might like to remove the stub from the StubToFunction map. - // We can't do that! Multiple threads could be stuck, waiting to acquire the - // lock above. As soon as the 1st function finishes compiling the function, - // the next one will be released, and needs to be able to find the function - // it needs to call. - //JR.state.getStubToFunctionMap(locked).erase(I); - - DOUT << "JIT: Lazily resolving function '" << F->getName() - << "' In stub ptr = " << Stub << " actual ptr = " - << ActualPtr << "\n"; - - Result = TheJIT->getPointerToFunction(F); + + DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName() + << "' In stub ptr = " << Stub << " actual ptr = " + << ActualPtr << "\n"); + (void)ActualPtr; + + Result = JR->TheJIT->getPointerToFunction(F); } - - // Reacquire the lock to erase the stub in the map. - MutexGuard locked(TheJIT->lock); - // We don't need to reuse this stub in the future, as F is now compiled. - JR.state.getFunctionToStubMap(locked).erase(F); + // Reacquire the lock to update the GOT map. + MutexGuard locked(JR->TheJIT->lock); + + // We might like to remove the call site from the CallSiteToFunction map, but + // we can't do that! Multiple threads could be stuck, waiting to acquire the + // lock above. As soon as the 1st function finishes compiling the function, + // the next one will be released, and needs to be able to find the function it + // needs to call. // FIXME: We could rewrite all references to this stub if we knew them. @@ -404,335 +674,17 @@ void *JITResolver::JITCompilerFn(void *Stub) { // if they see it still using the stub address. // Note: this is done so the Resolver doesn't have to manage GOT memory // Do this without allocating map space if the target isn't using a GOT - if(JR.revGOTMap.find(Stub) != JR.revGOTMap.end()) - JR.revGOTMap[Result] = JR.revGOTMap[Stub]; + if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end()) + JR->revGOTMap[Result] = JR->revGOTMap[Stub]; return Result; } -//===----------------------------------------------------------------------===// -// Function Index Support - -// On MacOS we generate an index of currently JIT'd functions so that -// performance tools can determine a symbol name and accurate code range for a -// PC value. Because performance tools are generally asynchronous, the code -// below is written with the hope that it could be interrupted at any time and -// have useful answers. However, we don't go crazy with atomic operations, we -// just do a "reasonable effort". -#ifdef __APPLE__ -#define ENABLE_JIT_SYMBOL_TABLE 0 -#endif - -/// JitSymbolEntry - Each function that is JIT compiled results in one of these -/// being added to an array of symbols. This indicates the name of the function -/// as well as the address range it occupies. This allows the client to map -/// from a PC value to the name of the function. -struct JitSymbolEntry { - const char *FnName; // FnName - a strdup'd string. - void *FnStart; - intptr_t FnSize; -}; - - -struct JitSymbolTable { - /// NextPtr - This forms a linked list of JitSymbolTable entries. This - /// pointer is not used right now, but might be used in the future. Consider - /// it reserved for future use. - JitSymbolTable *NextPtr; - - /// Symbols - This is an array of JitSymbolEntry entries. Only the first - /// 'NumSymbols' symbols are valid. - JitSymbolEntry *Symbols; - - /// NumSymbols - This indicates the number entries in the Symbols array that - /// are valid. - unsigned NumSymbols; - - /// NumAllocated - This indicates the amount of space we have in the Symbols - /// array. This is a private field that should not be read by external tools. - unsigned NumAllocated; -}; - -#if ENABLE_JIT_SYMBOL_TABLE -JitSymbolTable *__jitSymbolTable; -#endif - -static void AddFunctionToSymbolTable(const char *FnName, - void *FnStart, intptr_t FnSize) { - assert(FnName != 0 && FnStart != 0 && "Bad symbol to add"); - JitSymbolTable **SymTabPtrPtr = 0; -#if !ENABLE_JIT_SYMBOL_TABLE - return; -#else - SymTabPtrPtr = &__jitSymbolTable; -#endif - - // If this is the first entry in the symbol table, add the JitSymbolTable - // index. - if (*SymTabPtrPtr == 0) { - JitSymbolTable *New = new JitSymbolTable(); - New->NextPtr = 0; - New->Symbols = 0; - New->NumSymbols = 0; - New->NumAllocated = 0; - *SymTabPtrPtr = New; - } - - JitSymbolTable *SymTabPtr = *SymTabPtrPtr; - - // If we have space in the table, reallocate the table. - if (SymTabPtr->NumSymbols >= SymTabPtr->NumAllocated) { - // If we don't have space, reallocate the table. - unsigned NewSize = std::max(64U, SymTabPtr->NumAllocated*2); - JitSymbolEntry *NewSymbols = new JitSymbolEntry[NewSize]; - JitSymbolEntry *OldSymbols = SymTabPtr->Symbols; - - // Copy the old entries over. - memcpy(NewSymbols, OldSymbols, SymTabPtr->NumSymbols*sizeof(OldSymbols[0])); - - // Swap the new symbols in, delete the old ones. - SymTabPtr->Symbols = NewSymbols; - SymTabPtr->NumAllocated = NewSize; - delete [] OldSymbols; - } - - // Otherwise, we have enough space, just tack it onto the end of the array. - JitSymbolEntry &Entry = SymTabPtr->Symbols[SymTabPtr->NumSymbols]; - Entry.FnName = strdup(FnName); - Entry.FnStart = FnStart; - Entry.FnSize = FnSize; - ++SymTabPtr->NumSymbols; -} - -static void RemoveFunctionFromSymbolTable(void *FnStart) { - assert(FnStart && "Invalid function pointer"); - JitSymbolTable **SymTabPtrPtr = 0; -#if !ENABLE_JIT_SYMBOL_TABLE - return; -#else - SymTabPtrPtr = &__jitSymbolTable; -#endif - - JitSymbolTable *SymTabPtr = *SymTabPtrPtr; - JitSymbolEntry *Symbols = SymTabPtr->Symbols; - - // Scan the table to find its index. The table is not sorted, so do a linear - // scan. - unsigned Index; - for (Index = 0; Symbols[Index].FnStart != FnStart; ++Index) - assert(Index != SymTabPtr->NumSymbols && "Didn't find function!"); - - // Once we have an index, we know to nuke this entry, overwrite it with the - // entry at the end of the array, making the last entry redundant. - const char *OldName = Symbols[Index].FnName; - Symbols[Index] = Symbols[SymTabPtr->NumSymbols-1]; - free((void*)OldName); - - // Drop the number of symbols in the table. - --SymTabPtr->NumSymbols; - - // Finally, if we deleted the final symbol, deallocate the table itself. - if (SymTabPtr->NumSymbols != 0) - return; - - *SymTabPtrPtr = 0; - delete [] Symbols; - delete SymTabPtr; -} - //===----------------------------------------------------------------------===// // JITEmitter code. // -namespace { - /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is - /// used to output functions to memory for execution. - class JITEmitter : public JITCodeEmitter { - JITMemoryManager *MemMgr; - - // When outputting a function stub in the context of some other function, we - // save BufferBegin/BufferEnd/CurBufferPtr here. - unsigned char *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; - - /// Relocations - These are the relocations that the function needs, as - /// emitted. - std::vector Relocations; - - /// MBBLocations - This vector is a mapping from MBB ID's to their address. - /// It is filled in by the StartMachineBasicBlock callback and queried by - /// the getMachineBasicBlockAddress callback. - std::vector MBBLocations; - - /// ConstantPool - The constant pool for the current function. - /// - MachineConstantPool *ConstantPool; - - /// ConstantPoolBase - A pointer to the first entry in the constant pool. - /// - void *ConstantPoolBase; - - /// ConstPoolAddresses - Addresses of individual constant pool entries. - /// - SmallVector ConstPoolAddresses; - - /// JumpTable - The jump tables for the current function. - /// - MachineJumpTableInfo *JumpTable; - - /// JumpTableBase - A pointer to the first entry in the jump table. - /// - void *JumpTableBase; - - /// Resolver - This contains info about the currently resolved functions. - JITResolver Resolver; - - /// DE - The dwarf emitter for the jit. - JITDwarfEmitter *DE; - - /// LabelLocations - This vector is a mapping from Label ID's to their - /// address. - std::vector LabelLocations; - - /// MMI - Machine module info for exception informations - MachineModuleInfo* MMI; - - // GVSet - a set to keep track of which globals have been seen - SmallPtrSet GVSet; - - // CurFn - The llvm function being emitted. Only valid during - // finishFunction(). - const Function *CurFn; - - // CurFnStubUses - For a given Function, a vector of stubs that it - // references. This facilitates the JIT detecting that a stub is no - // longer used, so that it may be deallocated. - DenseMap > CurFnStubUses; - - // StubFnRefs - For a given pointer to a stub, a set of Functions which - // reference the stub. When the count of a stub's references drops to zero, - // the stub is unused. - DenseMap > StubFnRefs; - - // ExtFnStubs - A map of external function names to stubs which have entries - // in the JITResolver's ExternalFnToStubMap. - StringMap ExtFnStubs; - - // MCI - A pointer to a MachineCodeInfo object to update with information. - MachineCodeInfo *MCI; - - public: - JITEmitter(JIT &jit, JITMemoryManager *JMM) : Resolver(jit), CurFn(0), MCI(0) { - MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); - if (jit.getJITInfo().needsGOT()) { - MemMgr->AllocateGOT(); - DOUT << "JIT is managing a GOT\n"; - } - - if (ExceptionHandling) DE = new JITDwarfEmitter(jit); - } - ~JITEmitter() { - delete MemMgr; - if (ExceptionHandling) delete DE; - } - - /// classof - Methods for support type inquiry through isa, cast, and - /// dyn_cast: - /// - static inline bool classof(const JITEmitter*) { return true; } - static inline bool classof(const MachineCodeEmitter*) { return true; } - - JITResolver &getJITResolver() { return Resolver; } - - virtual void startFunction(MachineFunction &F); - virtual bool finishFunction(MachineFunction &F); - - void emitConstantPool(MachineConstantPool *MCP); - void initJumpTableInfo(MachineJumpTableInfo *MJTI); - void emitJumpTableInfo(MachineJumpTableInfo *MJTI); - - virtual void startGVStub(const GlobalValue* GV, unsigned StubSize, - unsigned Alignment = 1); - virtual void startGVStub(const GlobalValue* GV, void *Buffer, - unsigned StubSize); - virtual void* finishGVStub(const GlobalValue *GV); - - /// allocateSpace - Reserves space in the current block if any, or - /// allocate a new one of the given size. - virtual void *allocateSpace(uintptr_t Size, unsigned Alignment); - - virtual void addRelocation(const MachineRelocation &MR) { - Relocations.push_back(MR); - } - - virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) { - if (MBBLocations.size() <= (unsigned)MBB->getNumber()) - MBBLocations.resize((MBB->getNumber()+1)*2); - MBBLocations[MBB->getNumber()] = getCurrentPCValue(); - DOUT << "JIT: Emitting BB" << MBB->getNumber() << " at [" - << (void*) getCurrentPCValue() << "]\n"; - } - - virtual uintptr_t getConstantPoolEntryAddress(unsigned Entry) const; - virtual uintptr_t getJumpTableEntryAddress(unsigned Entry) const; - - virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const { - assert(MBBLocations.size() > (unsigned)MBB->getNumber() && - MBBLocations[MBB->getNumber()] && "MBB not emitted!"); - return MBBLocations[MBB->getNumber()]; - } - - /// deallocateMemForFunction - Deallocate all memory for the specified - /// function body. - void deallocateMemForFunction(Function *F); - - /// AddStubToCurrentFunction - Mark the current function being JIT'd as - /// using the stub at the specified address. Allows - /// deallocateMemForFunction to also remove stubs no longer referenced. - void AddStubToCurrentFunction(void *Stub); - - /// getExternalFnStubs - Accessor for the JIT to find stubs emitted for - /// MachineRelocations that reference external functions by name. - const StringMap &getExternalFnStubs() const { return ExtFnStubs; } - - virtual void emitLabel(uint64_t LabelID) { - if (LabelLocations.size() <= LabelID) - LabelLocations.resize((LabelID+1)*2); - LabelLocations[LabelID] = getCurrentPCValue(); - } - - virtual uintptr_t getLabelAddress(uint64_t LabelID) const { - assert(LabelLocations.size() > (unsigned)LabelID && - LabelLocations[LabelID] && "Label not emitted!"); - return LabelLocations[LabelID]; - } - - virtual void setModuleInfo(MachineModuleInfo* Info) { - MMI = Info; - if (ExceptionHandling) DE->setModuleInfo(Info); - } - - void setMemoryExecutable(void) { - MemMgr->setMemoryExecutable(); - } - - JITMemoryManager *getMemMgr(void) const { return MemMgr; } - - void setMachineCodeInfo(MachineCodeInfo *mci) { - MCI = mci; - } - - private: - void *getPointerToGlobal(GlobalValue *GV, void *Reference, bool NoNeedStub); - void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference, - bool NoNeedStub); - unsigned addSizeOfGlobal(const GlobalVariable *GV, unsigned Size); - unsigned addSizeOfGlobalsInConstantVal(const Constant *C, unsigned Size); - unsigned addSizeOfGlobalsInInitializer(const Constant *Init, unsigned Size); - unsigned GetSizeOfGlobalsInBytes(MachineFunction &MF); - }; -} - void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, - bool DoesntNeedStub) { + bool MayNeedFarStub) { if (GlobalVariable *GV = dyn_cast(V)) return TheJIT->getOrEmitGlobalVariable(GV); @@ -741,73 +693,61 @@ void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, // If we have already compiled the function, return a pointer to its body. Function *F = cast(V); - void *ResultPtr; - if (!DoesntNeedStub && !TheJIT->isLazyCompilationDisabled()) { - // Return the function stub if it's already created. - ResultPtr = Resolver.getFunctionStubIfAvailable(F); - if (ResultPtr) - AddStubToCurrentFunction(ResultPtr); - } else { - ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); + + void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F); + if (FnStub) { + // Return the function stub if it's already created. We do this first so + // that we're returning the same address for the function as any previous + // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be + // close enough to call. + return FnStub; } - if (ResultPtr) return ResultPtr; - - // If this is an external function pointer, we can force the JIT to - // 'compile' it, which really just adds it to the map. In dlsym mode, - // external functions are forced through a stub, regardless of reloc type. - if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode() && - DoesntNeedStub && !TheJIT->areDlsymStubsEnabled()) - return TheJIT->getPointerToFunction(F); - - // Okay, the function has not been compiled yet, if the target callback - // mechanism is capable of rewriting the instruction directly, prefer to do - // that instead of emitting a stub. This uses the lazy resolver, so is not - // legal if lazy compilation is disabled. - if (DoesntNeedStub && !TheJIT->isLazyCompilationDisabled()) - return Resolver.AddCallbackAtLocation(F, Reference); - - // Otherwise, we have to emit a stub. - void *StubAddr = Resolver.getFunctionStub(F); - - // Add the stub to the current function's list of referenced stubs, so we can - // deallocate them if the current function is ever freed. It's possible to - // return null from getFunctionStub in the case of a weak extern that fails - // to resolve. - if (StubAddr) - AddStubToCurrentFunction(StubAddr); - return StubAddr; + // If we know the target can handle arbitrary-distance calls, try to + // return a direct pointer. + if (!MayNeedFarStub) { + // If we have code, go ahead and return that. + void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); + if (ResultPtr) return ResultPtr; + + // If this is an external function pointer, we can force the JIT to + // 'compile' it, which really just adds it to the map. + if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) + return TheJIT->getPointerToFunction(F); + } + + // Otherwise, we may need a to emit a stub, and, conservatively, we always do + // so. Note that it's possible to return null from getLazyFunctionStub in the + // case of a weak extern that fails to resolve. + return Resolver.getLazyFunctionStub(F); } -void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference, - bool NoNeedStub) { +void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) { // Make sure GV is emitted first, and create a stub containing the fully // resolved address. - void *GVAddress = getPointerToGlobal(V, Reference, true); + void *GVAddress = getPointerToGlobal(V, Reference, false); void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress); - - // Add the stub to the current function's list of referenced stubs, so we can - // deallocate them if the current function is ever freed. - AddStubToCurrentFunction(StubAddr); - return StubAddr; } -void JITEmitter::AddStubToCurrentFunction(void *StubAddr) { - if (!TheJIT->areDlsymStubsEnabled()) - return; - - assert(CurFn && "Stub added to current function, but current function is 0!"); - - SmallVectorImpl &StubsUsed = CurFnStubUses[CurFn]; - StubsUsed.push_back(StubAddr); - - SmallPtrSet &FnRefs = StubFnRefs[StubAddr]; - FnRefs.insert(CurFn); +void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) { + if (DL.isUnknown()) return; + if (!BeforePrintingInsn) return; + + const LLVMContext &Context = EmissionDetails.MF->getFunction()->getContext(); + + if (DL.getScope(Context) != 0 && PrevDL != DL) { + JITEvent_EmittedFunctionDetails::LineStart NextLine; + NextLine.Address = getCurrentPCValue(); + NextLine.Loc = DL; + EmissionDetails.LineStarts.push_back(NextLine); + } + + PrevDL = DL; } static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, - const TargetData *TD) { + const DataLayout *TD) { const std::vector &Constants = MCP->getConstants(); if (Constants.empty()) return 0; @@ -816,251 +756,67 @@ static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, MachineConstantPoolEntry CPE = Constants[i]; unsigned AlignMask = CPE.getAlignment() - 1; Size = (Size + AlignMask) & ~AlignMask; - const Type *Ty = CPE.getType(); + Type *Ty = CPE.getType(); Size += TD->getTypeAllocSize(Ty); } return Size; } -static unsigned GetJumpTableSizeInBytes(MachineJumpTableInfo *MJTI) { - const std::vector &JT = MJTI->getJumpTables(); - if (JT.empty()) return 0; - - unsigned NumEntries = 0; - for (unsigned i = 0, e = JT.size(); i != e; ++i) - NumEntries += JT[i].MBBs.size(); - - unsigned EntrySize = MJTI->getEntrySize(); - - return NumEntries * EntrySize; -} - -static uintptr_t RoundUpToAlign(uintptr_t Size, unsigned Alignment) { - if (Alignment == 0) Alignment = 1; - // Since we do not know where the buffer will be allocated, be pessimistic. - return Size + Alignment; -} - -/// addSizeOfGlobal - add the size of the global (plus any alignment padding) -/// into the running total Size. - -unsigned JITEmitter::addSizeOfGlobal(const GlobalVariable *GV, unsigned Size) { - const Type *ElTy = GV->getType()->getElementType(); - size_t GVSize = (size_t)TheJIT->getTargetData()->getTypeAllocSize(ElTy); - size_t GVAlign = - (size_t)TheJIT->getTargetData()->getPreferredAlignment(GV); - DOUT << "JIT: Adding in size " << GVSize << " alignment " << GVAlign; - DEBUG(GV->dump()); - // Assume code section ends with worst possible alignment, so first - // variable needs maximal padding. - if (Size==0) - Size = 1; - Size = ((Size+GVAlign-1)/GVAlign)*GVAlign; - Size += GVSize; - return Size; -} - -/// addSizeOfGlobalsInConstantVal - find any globals that we haven't seen yet -/// but are referenced from the constant; put them in GVSet and add their -/// size into the running total Size. - -unsigned JITEmitter::addSizeOfGlobalsInConstantVal(const Constant *C, - unsigned Size) { - // If its undefined, return the garbage. - if (isa(C)) - return Size; - - // If the value is a ConstantExpr - if (const ConstantExpr *CE = dyn_cast(C)) { - Constant *Op0 = CE->getOperand(0); - switch (CE->getOpcode()) { - case Instruction::GetElementPtr: - case Instruction::Trunc: - case Instruction::ZExt: - case Instruction::SExt: - case Instruction::FPTrunc: - case Instruction::FPExt: - case Instruction::UIToFP: - case Instruction::SIToFP: - case Instruction::FPToUI: - case Instruction::FPToSI: - case Instruction::PtrToInt: - case Instruction::IntToPtr: - case Instruction::BitCast: { - Size = addSizeOfGlobalsInConstantVal(Op0, Size); - break; - } - case Instruction::Add: - case Instruction::Sub: - case Instruction::Mul: - case Instruction::UDiv: - case Instruction::SDiv: - case Instruction::URem: - case Instruction::SRem: - case Instruction::And: - case Instruction::Or: - case Instruction::Xor: { - Size = addSizeOfGlobalsInConstantVal(Op0, Size); - Size = addSizeOfGlobalsInConstantVal(CE->getOperand(1), Size); - break; - } - default: { - cerr << "ConstantExpr not handled: " << *CE << "\n"; - abort(); - } - } - } - - if (C->getType()->getTypeID() == Type::PointerTyID) - if (const GlobalVariable* GV = dyn_cast(C)) - if (GVSet.insert(GV)) - Size = addSizeOfGlobal(GV, Size); - - return Size; -} - -/// addSizeOfGLobalsInInitializer - handle any globals that we haven't seen yet -/// but are referenced from the given initializer. - -unsigned JITEmitter::addSizeOfGlobalsInInitializer(const Constant *Init, - unsigned Size) { - if (!isa(Init) && - !isa(Init) && - !isa(Init) && - !isa(Init) && - !isa(Init) && - Init->getType()->isFirstClassType()) - Size = addSizeOfGlobalsInConstantVal(Init, Size); - return Size; -} - -/// GetSizeOfGlobalsInBytes - walk the code for the function, looking for -/// globals; then walk the initializers of those globals looking for more. -/// If their size has not been considered yet, add it into the running total -/// Size. - -unsigned JITEmitter::GetSizeOfGlobalsInBytes(MachineFunction &MF) { - unsigned Size = 0; - GVSet.clear(); - - for (MachineFunction::iterator MBB = MF.begin(), E = MF.end(); - MBB != E; ++MBB) { - for (MachineBasicBlock::const_iterator I = MBB->begin(), E = MBB->end(); - I != E; ++I) { - const TargetInstrDesc &Desc = I->getDesc(); - const MachineInstr &MI = *I; - unsigned NumOps = Desc.getNumOperands(); - for (unsigned CurOp = 0; CurOp < NumOps; CurOp++) { - const MachineOperand &MO = MI.getOperand(CurOp); - if (MO.isGlobal()) { - GlobalValue* V = MO.getGlobal(); - const GlobalVariable *GV = dyn_cast(V); - if (!GV) - continue; - // If seen in previous function, it will have an entry here. - if (TheJIT->getPointerToGlobalIfAvailable(GV)) - continue; - // If seen earlier in this function, it will have an entry here. - // FIXME: it should be possible to combine these tables, by - // assuming the addresses of the new globals in this module - // start at 0 (or something) and adjusting them after codegen - // complete. Another possibility is to grab a marker bit in GV. - if (GVSet.insert(GV)) - // A variable as yet unseen. Add in its size. - Size = addSizeOfGlobal(GV, Size); - } - } - } - } - DOUT << "JIT: About to look through initializers\n"; - // Look for more globals that are referenced only from initializers. - // GVSet.end is computed each time because the set can grow as we go. - for (SmallPtrSet::iterator I = GVSet.begin(); - I != GVSet.end(); I++) { - const GlobalVariable* GV = *I; - if (GV->hasInitializer()) - Size = addSizeOfGlobalsInInitializer(GV->getInitializer(), Size); - } - - return Size; -} - void JITEmitter::startFunction(MachineFunction &F) { - DOUT << "JIT: Starting CodeGen of Function " - << F.getFunction()->getName() << "\n"; + DEBUG(dbgs() << "JIT: Starting CodeGen of Function " + << F.getName() << "\n"); uintptr_t ActualSize = 0; // Set the memory writable, if it's not already MemMgr->setMemoryWritable(); - if (MemMgr->NeedsExactSize()) { - DOUT << "JIT: ExactSize\n"; - const TargetInstrInfo* TII = F.getTarget().getInstrInfo(); - MachineJumpTableInfo *MJTI = F.getJumpTableInfo(); - MachineConstantPool *MCP = F.getConstantPool(); - - // Ensure the constant pool/jump table info is at least 4-byte aligned. - ActualSize = RoundUpToAlign(ActualSize, 16); - - // Add the alignment of the constant pool - ActualSize = RoundUpToAlign(ActualSize, MCP->getConstantPoolAlignment()); - - // Add the constant pool size - ActualSize += GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); - - // Add the aligment of the jump table info - ActualSize = RoundUpToAlign(ActualSize, MJTI->getAlignment()); - - // Add the jump table size - ActualSize += GetJumpTableSizeInBytes(MJTI); - - // Add the alignment for the function - ActualSize = RoundUpToAlign(ActualSize, - std::max(F.getFunction()->getAlignment(), 8U)); - - // Add the function size - ActualSize += TII->GetFunctionSizeInBytes(F); - - DOUT << "JIT: ActualSize before globals " << ActualSize << "\n"; - // Add the size of the globals that will be allocated after this function. - // These are all the ones referenced from this function that were not - // previously allocated. - ActualSize += GetSizeOfGlobalsInBytes(F); - DOUT << "JIT: ActualSize after globals " << ActualSize << "\n"; + + if (SizeEstimate > 0) { + // SizeEstimate will be non-zero on reallocation attempts. + ActualSize = SizeEstimate; } BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(), ActualSize); BufferEnd = BufferBegin+ActualSize; - + EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin; + // Ensure the constant pool/jump table info is at least 4-byte aligned. emitAlignment(16); emitConstantPool(F.getConstantPool()); - initJumpTableInfo(F.getJumpTableInfo()); + if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) + initJumpTableInfo(MJTI); // About to start emitting the machine code for the function. emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); + EmittedFunctions[F.getFunction()].Code = CurBufferPtr; MBBLocations.clear(); + + EmissionDetails.MF = &F; + EmissionDetails.LineStarts.clear(); } bool JITEmitter::finishFunction(MachineFunction &F) { if (CurBufferPtr == BufferEnd) { - // FIXME: Allocate more space, then try again. - cerr << "JIT: Ran out of space for generated machine code!\n"; - abort(); + // We must call endFunctionBody before retrying, because + // deallocateMemForFunction requires it. + MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); + retryWithMoreMemory(F); + return true; } - - emitJumpTableInfo(F.getJumpTableInfo()); - + + if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) + emitJumpTableInfo(MJTI); + // FnStart is the start of the text, not the start of the constant pool and // other per-function data. - unsigned char *FnStart = - (unsigned char *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); + uint8_t *FnStart = + (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); // FnEnd is the end of the function's machine code. - unsigned char *FnEnd = CurBufferPtr; + uint8_t *FnEnd = CurBufferPtr; if (!Relocations.empty()) { CurFn = F.getFunction(); @@ -1074,34 +830,25 @@ bool JITEmitter::finishFunction(MachineFunction &F) { if (MR.isExternalSymbol()) { ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(), false); - DOUT << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" - << ResultPtr << "]\n"; + DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" + << ResultPtr << "]\n"); // If the target REALLY wants a stub for this function, emit it now. - if (!MR.doesntNeedStub()) { - if (!TheJIT->areDlsymStubsEnabled()) { - ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); - } else { - void *&Stub = ExtFnStubs[MR.getExternalSymbol()]; - if (!Stub) { - Stub = Resolver.getExternalFunctionStub((void *)&Stub); - AddStubToCurrentFunction(Stub); - } - ResultPtr = Stub; - } + if (MR.mayNeedFarStub()) { + ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); } } else if (MR.isGlobalValue()) { ResultPtr = getPointerToGlobal(MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset(), - MR.doesntNeedStub()); + MR.mayNeedFarStub()); } else if (MR.isIndirectSymbol()) { - ResultPtr = getPointerToGVIndirectSym(MR.getGlobalValue(), - BufferBegin+MR.getMachineCodeOffset(), - MR.doesntNeedStub()); + ResultPtr = getPointerToGVIndirectSym( + MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset()); } else if (MR.isBasicBlock()) { ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); } else if (MR.isConstantPoolIndex()) { - ResultPtr = (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); + ResultPtr = + (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); } else { assert(MR.isJumpTableIndex()); ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); @@ -1116,9 +863,9 @@ bool JITEmitter::finishFunction(MachineFunction &F) { unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); MR.setGOTIndex(idx); if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { - DOUT << "JIT: GOT was out of date for " << ResultPtr - << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] - << "\n"; + DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr + << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] + << "\n"); ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; } } @@ -1133,8 +880,9 @@ bool JITEmitter::finishFunction(MachineFunction &F) { if (MemMgr->isManagingGOT()) { unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { - DOUT << "JIT: GOT was out of date for " << (void*)BufferBegin - << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] << "\n"; + DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin + << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] + << "\n"); ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; } } @@ -1144,9 +892,12 @@ bool JITEmitter::finishFunction(MachineFunction &F) { MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); if (CurBufferPtr == BufferEnd) { - // FIXME: Allocate more space, then try again. - cerr << "JIT: Ran out of space for generated machine code!\n"; - abort(); + retryWithMoreMemory(F); + return true; + } else { + // Now that we've succeeded in emitting the function, reset the + // SizeEstimate back down to zero. + SizeEstimate = 0; } BufferBegin = CurBufferPtr = 0; @@ -1154,20 +905,17 @@ bool JITEmitter::finishFunction(MachineFunction &F) { // Invalidate the icache if necessary. sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart); - - // Add it to the JIT symbol table if the host wants it. - AddFunctionToSymbolTable(F.getFunction()->getNameStart(), - FnStart, FnEnd-FnStart); - - DOUT << "JIT: Finished CodeGen of [" << (void*)FnStart - << "] Function: " << F.getFunction()->getName() - << ": " << (FnEnd-FnStart) << " bytes of text, " - << Relocations.size() << " relocations\n"; - - if (MCI) { - MCI->setAddress(FnStart); - MCI->setSize(FnEnd-FnStart); - } + + TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart, + EmissionDetails); + + // Reset the previous debug location. + PrevDL = DebugLoc(); + + DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart + << "] Function: " << F.getName() + << ": " << (FnEnd-FnStart) << " bytes of text, " + << Relocations.size() << " relocations\n"); Relocations.clear(); ConstPoolAddresses.clear(); @@ -1175,119 +923,73 @@ bool JITEmitter::finishFunction(MachineFunction &F) { // Mark code region readable and executable if it's not so already. MemMgr->setMemoryExecutable(); -#ifndef NDEBUG - { - if (sys::hasDisassembler()) { - DOUT << "JIT: Disassembled code:\n"; - DOUT << sys::disassembleBuffer(FnStart, FnEnd-FnStart, (uintptr_t)FnStart); - } else { - DOUT << "JIT: Binary code:\n"; - DOUT << std::hex; - unsigned char* q = FnStart; - for (int i = 0; q < FnEnd; q += 4, ++i) { - if (i == 4) - i = 0; - if (i == 0) - DOUT << "JIT: " << std::setw(8) << std::setfill('0') - << (long)(q - FnStart) << ": "; - bool Done = false; - for (int j = 3; j >= 0; --j) { - if (q + j >= FnEnd) - Done = true; - else - DOUT << std::setw(2) << std::setfill('0') << (unsigned short)q[j]; + DEBUG({ + if (sys::hasDisassembler()) { + dbgs() << "JIT: Disassembled code:\n"; + dbgs() << sys::disassembleBuffer(FnStart, FnEnd-FnStart, + (uintptr_t)FnStart); + } else { + dbgs() << "JIT: Binary code:\n"; + uint8_t* q = FnStart; + for (int i = 0; q < FnEnd; q += 4, ++i) { + if (i == 4) + i = 0; + if (i == 0) + dbgs() << "JIT: " << (long)(q - FnStart) << ": "; + bool Done = false; + for (int j = 3; j >= 0; --j) { + if (q + j >= FnEnd) + Done = true; + else + dbgs() << (unsigned short)q[j]; + } + if (Done) + break; + dbgs() << ' '; + if (i == 3) + dbgs() << '\n'; } - if (Done) - break; - DOUT << ' '; - if (i == 3) - DOUT << '\n'; + dbgs()<< '\n'; } - DOUT << std::dec; - DOUT<< '\n'; - } - } -#endif - if (ExceptionHandling) { - uintptr_t ActualSize = 0; - SavedBufferBegin = BufferBegin; - SavedBufferEnd = BufferEnd; - SavedCurBufferPtr = CurBufferPtr; - - if (MemMgr->NeedsExactSize()) { - ActualSize = DE->GetDwarfTableSizeInBytes(F, *this, FnStart, FnEnd); - } - - BufferBegin = CurBufferPtr = MemMgr->startExceptionTable(F.getFunction(), - ActualSize); - BufferEnd = BufferBegin+ActualSize; - unsigned char* FrameRegister = DE->EmitDwarfTable(F, *this, FnStart, FnEnd); - MemMgr->endExceptionTable(F.getFunction(), BufferBegin, CurBufferPtr, - FrameRegister); - BufferBegin = SavedBufferBegin; - BufferEnd = SavedBufferEnd; - CurBufferPtr = SavedCurBufferPtr; - - TheJIT->RegisterTable(FrameRegister); - } + }); if (MMI) MMI->EndFunction(); - + return false; } +void JITEmitter::retryWithMoreMemory(MachineFunction &F) { + DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n"); + Relocations.clear(); // Clear the old relocations or we'll reapply them. + ConstPoolAddresses.clear(); + ++NumRetries; + deallocateMemForFunction(F.getFunction()); + // Try again with at least twice as much free space. + SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin)); + + for (MachineFunction::iterator MBB = F.begin(), E = F.end(); MBB != E; ++MBB){ + if (MBB->hasAddressTaken()) + TheJIT->clearPointerToBasicBlock(MBB->getBasicBlock()); + } +} + /// deallocateMemForFunction - Deallocate all memory for the specified /// function body. Also drop any references the function has to stubs. -void JITEmitter::deallocateMemForFunction(Function *F) { - MemMgr->deallocateMemForFunction(F); - - // If the function did not reference any stubs, return. - if (CurFnStubUses.find(F) == CurFnStubUses.end()) - return; - - // For each referenced stub, erase the reference to this function, and then - // erase the list of referenced stubs. - SmallVectorImpl &StubList = CurFnStubUses[F]; - for (unsigned i = 0, e = StubList.size(); i != e; ++i) { - void *Stub = StubList[i]; - - // If we already invalidated this stub for this function, continue. - if (StubFnRefs.count(Stub) == 0) - continue; - - SmallPtrSet &FnRefs = StubFnRefs[Stub]; - FnRefs.erase(F); - - // If this function was the last reference to the stub, invalidate the stub - // in the JITResolver. Were there a memory manager deallocateStub routine, - // we could call that at this point too. - if (FnRefs.empty()) { - DOUT << "\nJIT: Invalidated Stub at [" << Stub << "]\n"; - StubFnRefs.erase(Stub); - - // Invalidate the stub. If it is a GV stub, update the JIT's global - // mapping for that GV to zero, otherwise, search the string map of - // external function names to stubs and remove the entry for this stub. - GlobalValue *GV = Resolver.invalidateStub(Stub); - if (GV) { - TheJIT->updateGlobalMapping(GV, 0); - } else { - for (StringMapIterator i = ExtFnStubs.begin(), - e = ExtFnStubs.end(); i != e; ++i) { - if (i->second == Stub) { - ExtFnStubs.erase(i); - break; - } - } - } - } +/// May be called while the Function is being destroyed inside ~Value(). +void JITEmitter::deallocateMemForFunction(const Function *F) { + ValueMap::iterator + Emitted = EmittedFunctions.find(F); + if (Emitted != EmittedFunctions.end()) { + MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody); + TheJIT->NotifyFreeingMachineCode(Emitted->second.Code); + + EmittedFunctions.erase(Emitted); } - CurFnStubUses.erase(F); } -void* JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { +void *JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { if (BufferBegin) return JITCodeEmitter::allocateSpace(Size, Alignment); @@ -1299,6 +1001,11 @@ void* JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { return CurBufferPtr; } +void *JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { + // Delegate this call through the memory manager. + return MemMgr->allocateGlobal(Size, Alignment); +} + void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { if (TheJIT->getJITInfo().hasCustomConstantPool()) return; @@ -1306,15 +1013,15 @@ void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { const std::vector &Constants = MCP->getConstants(); if (Constants.empty()) return; - unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); + unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getDataLayout()); unsigned Align = MCP->getConstantPoolAlignment(); ConstantPoolBase = allocateSpace(Size, Align); ConstantPool = MCP; if (ConstantPoolBase == 0) return; // Buffer overflow. - DOUT << "JIT: Emitted constant pool at [" << ConstantPoolBase - << "] (size: " << Size << ", alignment: " << Align << ")\n"; + DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase + << "] (size: " << Size << ", alignment: " << Align << ")\n"); // Initialize the memory for all of the constant pool entries. unsigned Offset = 0; @@ -1327,37 +1034,39 @@ void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { ConstPoolAddresses.push_back(CAddr); if (CPE.isMachineConstantPoolEntry()) { // FIXME: add support to lower machine constant pool values into bytes! - cerr << "Initialize memory with machine specific constant pool entry" - << " has not been implemented!\n"; - abort(); + report_fatal_error("Initialize memory with machine specific constant pool" + "entry has not been implemented!"); } TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr); - DOUT << "JIT: CP" << i << " at [0x" - << std::hex << CAddr << std::dec << "]\n"; + DEBUG(dbgs() << "JIT: CP" << i << " at [0x"; + dbgs().write_hex(CAddr) << "]\n"); - const Type *Ty = CPE.Val.ConstVal->getType(); - Offset += TheJIT->getTargetData()->getTypeAllocSize(Ty); + Type *Ty = CPE.Val.ConstVal->getType(); + Offset += TheJIT->getDataLayout()->getTypeAllocSize(Ty); } } void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { if (TheJIT->getJITInfo().hasCustomJumpTables()) return; + if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) + return; const std::vector &JT = MJTI->getJumpTables(); if (JT.empty()) return; - + unsigned NumEntries = 0; for (unsigned i = 0, e = JT.size(); i != e; ++i) NumEntries += JT[i].MBBs.size(); - unsigned EntrySize = MJTI->getEntrySize(); + unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getDataLayout()); // Just allocate space for all the jump tables now. We will fix up the actual // MBB entries in the tables after we emit the code for each block, since then // we will know the final locations of the MBBs in memory. JumpTable = MJTI; - JumpTableBase = allocateSpace(NumEntries * EntrySize, MJTI->getAlignment()); + JumpTableBase = allocateSpace(NumEntries * EntrySize, + MJTI->getEntryAlignment(*TheJIT->getDataLayout())); } void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { @@ -1366,9 +1075,35 @@ void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { const std::vector &JT = MJTI->getJumpTables(); if (JT.empty() || JumpTableBase == 0) return; - - if (TargetMachine::getRelocationModel() == Reloc::PIC_) { - assert(MJTI->getEntrySize() == 4 && "Cross JIT'ing?"); + + + switch (MJTI->getEntryKind()) { + case MachineJumpTableInfo::EK_Inline: + return; + case MachineJumpTableInfo::EK_BlockAddress: { + // EK_BlockAddress - Each entry is a plain address of block, e.g.: + // .word LBB123 + assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == sizeof(void*) && + "Cross JIT'ing?"); + + // For each jump table, map each target in the jump table to the address of + // an emitted MachineBasicBlock. + intptr_t *SlotPtr = (intptr_t*)JumpTableBase; + + for (unsigned i = 0, e = JT.size(); i != e; ++i) { + const std::vector &MBBs = JT[i].MBBs; + // Store the address of the basic block for this jump table slot in the + // memory we allocated for the jump table in 'initJumpTableInfo' + for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) + *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); + } + break; + } + + case MachineJumpTableInfo::EK_Custom32: + case MachineJumpTableInfo::EK_GPRel32BlockAddress: + case MachineJumpTableInfo::EK_LabelDifference32: { + assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == 4&&"Cross JIT'ing?"); // For each jump table, place the offset from the beginning of the table // to the target address. int *SlotPtr = (int*)JumpTableBase; @@ -1380,52 +1115,51 @@ void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { uintptr_t Base = (uintptr_t)SlotPtr; for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); + /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook. *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); } } - } else { - assert(MJTI->getEntrySize() == sizeof(void*) && "Cross JIT'ing?"); - - // For each jump table, map each target in the jump table to the address of - // an emitted MachineBasicBlock. - intptr_t *SlotPtr = (intptr_t*)JumpTableBase; - - for (unsigned i = 0, e = JT.size(); i != e; ++i) { - const std::vector &MBBs = JT[i].MBBs; - // Store the address of the basic block for this jump table slot in the - // memory we allocated for the jump table in 'initJumpTableInfo' - for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) - *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); - } + break; + } + case MachineJumpTableInfo::EK_GPRel64BlockAddress: + llvm_unreachable( + "JT Info emission not implemented for GPRel64BlockAddress yet."); } } -void JITEmitter::startGVStub(const GlobalValue* GV, unsigned StubSize, - unsigned Alignment) { +void JITEmitter::startGVStub(const GlobalValue* GV, + unsigned StubSize, unsigned Alignment) { SavedBufferBegin = BufferBegin; SavedBufferEnd = BufferEnd; SavedCurBufferPtr = CurBufferPtr; - + BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); BufferEnd = BufferBegin+StubSize+1; } -void JITEmitter::startGVStub(const GlobalValue* GV, void *Buffer, - unsigned StubSize) { +void JITEmitter::startGVStub(void *Buffer, unsigned StubSize) { SavedBufferBegin = BufferBegin; SavedBufferEnd = BufferEnd; SavedCurBufferPtr = CurBufferPtr; - - BufferBegin = CurBufferPtr = (unsigned char *)Buffer; + + BufferBegin = CurBufferPtr = (uint8_t *)Buffer; BufferEnd = BufferBegin+StubSize+1; } -void *JITEmitter::finishGVStub(const GlobalValue* GV) { +void JITEmitter::finishGVStub() { + assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space."); NumBytes += getCurrentPCOffset(); - std::swap(SavedBufferBegin, BufferBegin); + BufferBegin = SavedBufferBegin; BufferEnd = SavedBufferEnd; CurBufferPtr = SavedCurBufferPtr; - return SavedBufferBegin; +} + +void *JITEmitter::allocIndirectGV(const GlobalValue *GV, + const uint8_t *Buffer, size_t Size, + unsigned Alignment) { + uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment); + memcpy(IndGV, Buffer, Size); + return IndGV; } // getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry @@ -1444,37 +1178,36 @@ uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { const std::vector &JT = JumpTable->getJumpTables(); assert(Index < JT.size() && "Invalid jump table index!"); - + + unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getDataLayout()); + unsigned Offset = 0; - unsigned EntrySize = JumpTable->getEntrySize(); - for (unsigned i = 0; i < Index; ++i) Offset += JT[i].MBBs.size(); - + Offset *= EntrySize; - + return (uintptr_t)((char *)JumpTableBase + Offset); } +void JITEmitter::EmittedFunctionConfig::onDelete( + JITEmitter *Emitter, const Function *F) { + Emitter->deallocateMemForFunction(F); +} +void JITEmitter::EmittedFunctionConfig::onRAUW( + JITEmitter *, const Function*, const Function*) { + llvm_unreachable("The JIT doesn't know how to handle a" + " RAUW on a value it has emitted."); +} + + //===----------------------------------------------------------------------===// // Public interface to this file //===----------------------------------------------------------------------===// -JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM) { - return new JITEmitter(jit, JMM); -} - -// getPointerToNamedFunction - This function is used as a global wrapper to -// JIT::getPointerToNamedFunction for the purpose of resolving symbols when -// bugpoint is debugging the JIT. In that scenario, we are loading an .so and -// need to resolve function(s) that are being mis-codegenerated, so we need to -// resolve their addresses at runtime, and this is the way to do it. -extern "C" { - void *getPointerToNamedFunction(const char *Name) { - if (Function *F = TheJIT->FindFunctionNamed(Name)) - return TheJIT->getPointerToFunction(F); - return TheJIT->getPointerToNamedFunction(Name); - } +JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM, + TargetMachine &tm) { + return new JITEmitter(jit, JMM, tm); } // getPointerToFunctionOrStub - If the specified function has been @@ -1485,131 +1218,34 @@ void *JIT::getPointerToFunctionOrStub(Function *F) { // If we have already code generated the function, just return the address. if (void *Addr = getPointerToGlobalIfAvailable(F)) return Addr; - - // Get a stub if the target supports it. - assert(isa(JCE) && "Unexpected MCE?"); - JITEmitter *JE = cast(getCodeEmitter()); - return JE->getJITResolver().getFunctionStub(F); -} -void JIT::registerMachineCodeInfo(MachineCodeInfo *mc) { - assert(isa(JCE) && "Unexpected MCE?"); - JITEmitter *JE = cast(getCodeEmitter()); - - JE->setMachineCodeInfo(mc); + // Get a stub if the target supports it. + JITEmitter *JE = static_cast(getCodeEmitter()); + return JE->getJITResolver().getLazyFunctionStub(F); } void JIT::updateFunctionStub(Function *F) { // Get the empty stub we generated earlier. - assert(isa(JCE) && "Unexpected MCE?"); - JITEmitter *JE = cast(getCodeEmitter()); - void *Stub = JE->getJITResolver().getFunctionStub(F); + JITEmitter *JE = static_cast(getCodeEmitter()); + void *Stub = JE->getJITResolver().getLazyFunctionStub(F); + void *Addr = getPointerToGlobalIfAvailable(F); + assert(Addr != Stub && "Function must have non-stub address to be updated."); // Tell the target jit info to rewrite the stub at the specified address, // rather than creating a new one. - void *Addr = getPointerToGlobalIfAvailable(F); - getJITInfo().emitFunctionStubAtAddr(F, Addr, Stub, *getCodeEmitter()); -} - -/// updateDlsymStubTable - Emit the data necessary to relocate the stubs -/// that were emitted during code generation. -/// -void JIT::updateDlsymStubTable() { - assert(isa(JCE) && "Unexpected MCE?"); - JITEmitter *JE = cast(getCodeEmitter()); - - SmallVector GVs; - SmallVector Ptrs; - const StringMap &ExtFns = JE->getExternalFnStubs(); - - JE->getJITResolver().getRelocatableGVs(GVs, Ptrs); - - unsigned nStubs = GVs.size() + ExtFns.size(); - - // If there are no relocatable stubs, return. - if (nStubs == 0) - return; - - // If there are no new relocatable stubs, return. - void *CurTable = JE->getMemMgr()->getDlsymTable(); - if (CurTable && (*(unsigned *)CurTable == nStubs)) - return; - - // Calculate the size of the stub info - unsigned offset = 4 + 4 * nStubs + sizeof(intptr_t) * nStubs; - - SmallVector Offsets; - for (unsigned i = 0; i != GVs.size(); ++i) { - Offsets.push_back(offset); - offset += GVs[i]->getName().length() + 1; - } - for (StringMapConstIterator i = ExtFns.begin(), e = ExtFns.end(); - i != e; ++i) { - Offsets.push_back(offset); - offset += strlen(i->first()) + 1; - } - - // Allocate space for the new "stub", which contains the dlsym table. - JE->startGVStub(0, offset, 4); - - // Emit the number of records - JE->emitInt32(nStubs); - - // Emit the string offsets - for (unsigned i = 0; i != nStubs; ++i) - JE->emitInt32(Offsets[i]); - - // Emit the pointers. Verify that they are at least 2-byte aligned, and set - // the low bit to 0 == GV, 1 == Function, so that the client code doing the - // relocation can write the relocated pointer at the appropriate place in - // the stub. - for (unsigned i = 0; i != GVs.size(); ++i) { - intptr_t Ptr = (intptr_t)Ptrs[i]; - assert((Ptr & 1) == 0 && "Stub pointers must be at least 2-byte aligned!"); - - if (isa(GVs[i])) - Ptr |= (intptr_t)1; - - if (sizeof(Ptr) == 8) - JE->emitInt64(Ptr); - else - JE->emitInt32(Ptr); - } - for (StringMapConstIterator i = ExtFns.begin(), e = ExtFns.end(); - i != e; ++i) { - intptr_t Ptr = (intptr_t)i->second | 1; - - if (sizeof(Ptr) == 8) - JE->emitInt64(Ptr); - else - JE->emitInt32(Ptr); - } - - // Emit the strings. - for (unsigned i = 0; i != GVs.size(); ++i) - JE->emitString(GVs[i]->getName()); - for (StringMapConstIterator i = ExtFns.begin(), e = ExtFns.end(); - i != e; ++i) - JE->emitString(i->first()); - - // Tell the JIT memory manager where it is. The JIT Memory Manager will - // deallocate space for the old one, if one existed. - JE->getMemMgr()->SetDlsymTable(JE->finishGVStub(0)); + TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout(); + JE->startGVStub(Stub, layout.Size); + getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter()); + JE->finishGVStub(); } /// freeMachineCodeForFunction - release machine code memory for given Function. /// void JIT::freeMachineCodeForFunction(Function *F) { - // Delete translation for this from the ExecutionEngine, so it will get // retranslated next time it is used. - void *OldPtr = updateGlobalMapping(F, 0); - - if (OldPtr) - RemoveFunctionFromSymbolTable(OldPtr); + updateGlobalMapping(F, 0); // Free the actual memory for the function body and related stuff. - assert(isa(JCE) && "Unexpected MCE?"); - cast(JCE)->deallocateMemForFunction(F); + static_cast(JCE)->deallocateMemForFunction(F); } -