X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FExecutionEngine%2FJIT%2FJITMemoryManager.cpp;h=bd0519e9c46435058fe2977c3b282203af14ef81;hb=53608a34ce3f0969e9fb01eaa983422761011e03;hp=688e486bed0b3cbe459da276ebbc427a414d1919;hpb=6f348e458660063a40052b208bab96895c822877;p=oota-llvm.git diff --git a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp b/lib/ExecutionEngine/JIT/JITMemoryManager.cpp index 688e486bed0..bd0519e9c46 100644 --- a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp +++ b/lib/ExecutionEngine/JIT/JITMemoryManager.cpp @@ -15,20 +15,30 @@ #include "llvm/ExecutionEngine/JITMemoryManager.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/Twine.h" #include "llvm/GlobalValue.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/System/Memory.h" -#include +#include "llvm/Support/Memory.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Config/config.h" #include #include #include -#include -#include #include + +#if defined(__linux__) +#if defined(HAVE_SYS_STAT_H) +#include +#endif +#include +#include +#endif + using namespace llvm; STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT"); @@ -49,23 +59,23 @@ namespace { /// ThisAllocated - This is true if this block is currently allocated. If /// not, this can be converted to a FreeRangeHeader. unsigned ThisAllocated : 1; - + /// PrevAllocated - Keep track of whether the block immediately before us is /// allocated. If not, the word immediately before this header is the size /// of the previous block. unsigned PrevAllocated : 1; - + /// BlockSize - This is the size in bytes of this memory block, /// including this header. uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2); - + /// getBlockAfter - Return the memory block immediately after this one. /// MemoryRangeHeader &getBlockAfter() const { return *(MemoryRangeHeader*)((char*)this+BlockSize); } - + /// getFreeBlockBefore - If the block before this one is free, return it, /// otherwise return null. FreeRangeHeader *getFreeBlockBefore() const { @@ -73,15 +83,15 @@ namespace { intptr_t PrevSize = ((intptr_t *)this)[-1]; return (FreeRangeHeader*)((char*)this-PrevSize); } - + /// FreeBlock - Turn an allocated block into a free block, adjusting /// bits in the object headers, and adding an end of region memory block. FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList); - + /// TrimAllocationToSize - If this allocated block is significantly larger /// than NewSize, split it into two pieces (where the former is NewSize /// bytes, including the header), and add the new block to the free list. - FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, + FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize); }; @@ -91,13 +101,13 @@ namespace { struct FreeRangeHeader : public MemoryRangeHeader { FreeRangeHeader *Prev; FreeRangeHeader *Next; - + /// getMinBlockSize - Get the minimum size for a memory block. Blocks /// smaller than this size cannot be created. static unsigned getMinBlockSize() { return sizeof(FreeRangeHeader)+sizeof(intptr_t); } - + /// SetEndOfBlockSizeMarker - The word at the end of every free block is /// known to be the size of the free block. Set it for this block. void SetEndOfBlockSizeMarker() { @@ -110,7 +120,7 @@ namespace { Next->Prev = Prev; return Prev->Next = Next; } - + void AddToFreeList(FreeRangeHeader *FreeList) { Next = FreeList; Prev = FreeList->Prev; @@ -121,7 +131,7 @@ namespace { /// GrowBlock - The block after this block just got deallocated. Merge it /// into the current block. void GrowBlock(uintptr_t NewSize); - + /// AllocateBlock - Mark this entire block allocated, updating freelists /// etc. This returns a pointer to the circular free-list. FreeRangeHeader *AllocateBlock(); @@ -137,7 +147,7 @@ FreeRangeHeader *FreeRangeHeader::AllocateBlock() { // Mark this block allocated. ThisAllocated = 1; getBlockAfter().PrevAllocated = 1; - + // Remove it from the free list. return RemoveFromFreeList(); } @@ -150,9 +160,9 @@ FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { MemoryRangeHeader *FollowingBlock = &getBlockAfter(); assert(ThisAllocated && "This block is already free!"); assert(FollowingBlock->PrevAllocated && "Flags out of sync!"); - + FreeRangeHeader *FreeListToReturn = FreeList; - + // If the block after this one is free, merge it into this block. if (!FollowingBlock->ThisAllocated) { FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock; @@ -164,18 +174,18 @@ FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { assert(&FollowingFreeBlock != FreeList && "No tombstone block?"); } FollowingFreeBlock.RemoveFromFreeList(); - + // Include the following block into this one. BlockSize += FollowingFreeBlock.BlockSize; FollowingBlock = &FollowingFreeBlock.getBlockAfter(); - + // Tell the block after the block we are coalescing that this block is // allocated. FollowingBlock->PrevAllocated = 1; } - + assert(FollowingBlock->ThisAllocated && "Missed coalescing?"); - + if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) { PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize); return FreeListToReturn ? FreeListToReturn : PrevFreeBlock; @@ -218,24 +228,24 @@ TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) { // Round up size for alignment of header. unsigned HeaderAlign = __alignof(FreeRangeHeader); NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1); - + // Size is now the size of the block we will remove from the start of the // current block. assert(NewSize <= BlockSize && "Allocating more space from this block than exists!"); - + // If splitting this block will cause the remainder to be too small, do not // split the block. if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize()) return FreeList; - + // Otherwise, we splice the required number of bytes out of this block, form // a new block immediately after it, then mark this block allocated. MemoryRangeHeader &FormerNextBlock = getBlockAfter(); - + // Change the size of this block. BlockSize = NewSize; - + // Get the new block we just sliced out and turn it into a free block. FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter(); NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock; @@ -283,7 +293,7 @@ namespace { sys::MemoryBlock LastSlab; // Memory slabs allocated by the JIT. We refer to them as slabs so we don't - // confuse them with the blocks of memory descibed above. + // confuse them with the blocks of memory described above. std::vector CodeSlabs; JITSlabAllocator BumpSlabAllocator; BumpPtrAllocator StubAllocator; @@ -316,6 +326,11 @@ namespace { /// should allocate a separate slab. static const size_t DefaultSizeThreshold; + /// getPointerToNamedFunction - This method returns the address of the + /// specified function by using the dlsym function call. + virtual void *getPointerToNamedFunction(const std::string &Name, + bool AbortOnFailure = true); + void AllocateGOT(); // Testing methods. @@ -347,12 +362,12 @@ namespace { } largest = largest - sizeof(MemoryRangeHeader); - + // If this block isn't big enough for the allocation desired, allocate // another block of memory and add it to the free list. if (largest < ActualSize || largest <= FreeRangeHeader::getMinBlockSize()) { - DEBUG(errs() << "JIT: Allocating another slab of memory for function."); + DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); candidateBlock = allocateNewCodeSlab((size_t)ActualSize); } @@ -443,30 +458,81 @@ namespace { return (uint8_t*)DataAllocator.Allocate(Size, Alignment); } - /// startExceptionTable - Use startFunctionBody to allocate memory for the + /// allocateCodeSection - Allocate memory for a code section. + uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID) { + // Grow the required block size to account for the block header + Size += sizeof(*CurBlock); + + // FIXME: Alignement handling. + FreeRangeHeader* candidateBlock = FreeMemoryList; + FreeRangeHeader* head = FreeMemoryList; + FreeRangeHeader* iter = head->Next; + + uintptr_t largest = candidateBlock->BlockSize; + + // Search for the largest free block. + while (iter != head) { + if (iter->BlockSize > largest) { + largest = iter->BlockSize; + candidateBlock = iter; + } + iter = iter->Next; + } + + largest = largest - sizeof(MemoryRangeHeader); + + // If this block isn't big enough for the allocation desired, allocate + // another block of memory and add it to the free list. + if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) { + DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); + candidateBlock = allocateNewCodeSlab((size_t)Size); + } + + // Select this candidate block for allocation + CurBlock = candidateBlock; + + // Allocate the entire memory block. + FreeMemoryList = candidateBlock->AllocateBlock(); + // Release the memory at the end of this block that isn't needed. + FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size); + return (uint8_t *)(CurBlock + 1); + } + + /// allocateDataSection - Allocate memory for a data section. + uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, bool IsReadOnly) { + return (uint8_t*)DataAllocator.Allocate(Size, Alignment); + } + + bool applyPermissions(std::string *ErrMsg) { + return false; + } + + /// startExceptionTable - Use startFunctionBody to allocate memory for the /// function's exception table. uint8_t* startExceptionTable(const Function* F, uintptr_t &ActualSize) { return startFunctionBody(F, ActualSize); } - /// endExceptionTable - The exception table of F is now allocated, + /// endExceptionTable - The exception table of F is now allocated, /// and takes the memory in the range [TableStart,TableEnd). void endExceptionTable(const Function *F, uint8_t *TableStart, uint8_t *TableEnd, uint8_t* FrameRegister) { assert(TableEnd > TableStart); assert(TableStart == (uint8_t *)(CurBlock+1) && "Mismatched table start/end!"); - + uintptr_t BlockSize = TableEnd - (uint8_t *)CurBlock; // Release the memory at the end of this block that isn't needed. FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); } - + uint8_t *getGOTBase() const { return GOTBase; } - + void deallocateBlock(void *Block) { // Find the block that is allocated for this function. MemoryRangeHeader *MemRange = static_cast(Block) - 1; @@ -555,16 +621,16 @@ DefaultJITMemoryManager::DefaultJITMemoryManager() // END ] // // The last three blocks are never deallocated or touched. - + // Add MemoryRangeHeader to the end of the memory region, indicating that // the space after the block of memory is allocated. This is block #3. MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1; Mem3->ThisAllocated = 1; Mem3->PrevAllocated = 0; Mem3->BlockSize = sizeof(MemoryRangeHeader); - + /// Add a tiny free region so that the free list always has one entry. - FreeRangeHeader *Mem2 = + FreeRangeHeader *Mem2 = (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize()); Mem2->ThisAllocated = 0; Mem2->PrevAllocated = 1; @@ -578,7 +644,7 @@ DefaultJITMemoryManager::DefaultJITMemoryManager() Mem1->ThisAllocated = 1; Mem1->PrevAllocated = 0; Mem1->BlockSize = sizeof(MemoryRangeHeader); - + // Add a FreeRangeHeader to the start of the function body region, indicating // that the space is free. Mark the previous block allocated so we never look // at it. @@ -588,7 +654,7 @@ DefaultJITMemoryManager::DefaultJITMemoryManager() Mem0->BlockSize = (char*)Mem1-(char*)Mem0; Mem0->SetEndOfBlockSizeMarker(); Mem0->AddToFreeList(Mem2); - + // Start out with the freelist pointing to Mem0. FreeMemoryList = Mem0; @@ -614,8 +680,8 @@ sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) { sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : 0; sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg); if (B.base() == 0) { - llvm_report_error("Allocation failed when allocating new memory in the" - " JIT\n" + ErrMsg); + report_fatal_error("Allocation failed when allocating new memory in the" + " JIT\n" + Twine(ErrMsg)); } LastSlab = B; ++NumSlabs; @@ -715,6 +781,139 @@ bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) { return true; } +//===----------------------------------------------------------------------===// +// getPointerToNamedFunction() implementation. +//===----------------------------------------------------------------------===// + +// AtExitHandlers - List of functions to call when the program exits, +// registered with the atexit() library function. +static std::vector AtExitHandlers; + +/// runAtExitHandlers - Run any functions registered by the program's +/// calls to atexit(3), which we intercept and store in +/// AtExitHandlers. +/// +static void runAtExitHandlers() { + while (!AtExitHandlers.empty()) { + void (*Fn)() = AtExitHandlers.back(); + AtExitHandlers.pop_back(); + Fn(); + } +} + +//===----------------------------------------------------------------------===// +// Function stubs that are invoked instead of certain library calls +// +// Force the following functions to be linked in to anything that uses the +// JIT. This is a hack designed to work around the all-too-clever Glibc +// strategy of making these functions work differently when inlined vs. when +// not inlined, and hiding their real definitions in a separate archive file +// that the dynamic linker can't see. For more info, search for +// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274. +#if defined(__linux__) +/* stat functions are redirecting to __xstat with a version number. On x86-64 + * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat' + * available as an exported symbol, so we have to add it explicitly. + */ +namespace { +class StatSymbols { +public: + StatSymbols() { + sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat); + sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat); + sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat); + sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64); + sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64); + sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64); + sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64); + sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64); + sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64); + sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit); + sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod); + } +}; +} +static StatSymbols initStatSymbols; +#endif // __linux__ + +// jit_exit - Used to intercept the "exit" library call. +static void jit_exit(int Status) { + runAtExitHandlers(); // Run atexit handlers... + exit(Status); +} + +// jit_atexit - Used to intercept the "atexit" library call. +static int jit_atexit(void (*Fn)()) { + AtExitHandlers.push_back(Fn); // Take note of atexit handler... + return 0; // Always successful +} + +static int jit_noop() { + return 0; +} + +//===----------------------------------------------------------------------===// +// +/// getPointerToNamedFunction - This method returns the address of the specified +/// function by using the dynamic loader interface. As such it is only useful +/// for resolving library symbols, not code generated symbols. +/// +void *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name, + bool AbortOnFailure) { + // Check to see if this is one of the functions we want to intercept. Note, + // we cast to intptr_t here to silence a -pedantic warning that complains + // about casting a function pointer to a normal pointer. + if (Name == "exit") return (void*)(intptr_t)&jit_exit; + if (Name == "atexit") return (void*)(intptr_t)&jit_atexit; + + // We should not invoke parent's ctors/dtors from generated main()! + // On Mingw and Cygwin, the symbol __main is resolved to + // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors + // (and register wrong callee's dtors with atexit(3)). + // We expect ExecutionEngine::runStaticConstructorsDestructors() + // is called before ExecutionEngine::runFunctionAsMain() is called. + if (Name == "__main") return (void*)(intptr_t)&jit_noop; + + const char *NameStr = Name.c_str(); + // If this is an asm specifier, skip the sentinal. + if (NameStr[0] == 1) ++NameStr; + + // If it's an external function, look it up in the process image... + void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr); + if (Ptr) return Ptr; + + // If it wasn't found and if it starts with an underscore ('_') character, + // try again without the underscore. + if (NameStr[0] == '_') { + Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1); + if (Ptr) return Ptr; + } + + // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These + // are references to hidden visibility symbols that dlsym cannot resolve. + // If we have one of these, strip off $LDBLStub and try again. +#if defined(__APPLE__) && defined(__ppc__) + if (Name.size() > 9 && Name[Name.size()-9] == '$' && + memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) { + // First try turning $LDBLStub into $LDBL128. If that fails, strip it off. + // This mirrors logic in libSystemStubs.a. + std::string Prefix = std::string(Name.begin(), Name.end()-9); + if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false)) + return Ptr; + if (void *Ptr = getPointerToNamedFunction(Prefix, false)) + return Ptr; + } +#endif + + if (AbortOnFailure) { + report_fatal_error("Program used external function '"+Name+ + "' which could not be resolved!"); + } + return 0; +} + + + JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() { return new DefaultJITMemoryManager(); }