1 //===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the DefaultJITMemoryManager class.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/GlobalValue.h"
15 #include "llvm/ExecutionEngine/JITMemoryManager.h"
16 #include "llvm/Support/Compiler.h"
17 #include "llvm/Support/ErrorHandling.h"
18 #include "llvm/System/Memory.h"
29 JITMemoryManager::~JITMemoryManager() {}
31 //===----------------------------------------------------------------------===//
32 // Memory Block Implementation.
33 //===----------------------------------------------------------------------===//
36 /// MemoryRangeHeader - For a range of memory, this is the header that we put
37 /// on the block of memory. It is carefully crafted to be one word of memory.
38 /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader
39 /// which starts with this.
40 struct FreeRangeHeader;
41 struct MemoryRangeHeader {
42 /// ThisAllocated - This is true if this block is currently allocated. If
43 /// not, this can be converted to a FreeRangeHeader.
44 unsigned ThisAllocated : 1;
46 /// PrevAllocated - Keep track of whether the block immediately before us is
47 /// allocated. If not, the word immediately before this header is the size
48 /// of the previous block.
49 unsigned PrevAllocated : 1;
51 /// BlockSize - This is the size in bytes of this memory block,
52 /// including this header.
53 uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2);
56 /// getBlockAfter - Return the memory block immediately after this one.
58 MemoryRangeHeader &getBlockAfter() const {
59 return *(MemoryRangeHeader*)((char*)this+BlockSize);
62 /// getFreeBlockBefore - If the block before this one is free, return it,
63 /// otherwise return null.
64 FreeRangeHeader *getFreeBlockBefore() const {
65 if (PrevAllocated) return 0;
66 intptr_t PrevSize = ((intptr_t *)this)[-1];
67 return (FreeRangeHeader*)((char*)this-PrevSize);
70 /// FreeBlock - Turn an allocated block into a free block, adjusting
71 /// bits in the object headers, and adding an end of region memory block.
72 FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList);
74 /// TrimAllocationToSize - If this allocated block is significantly larger
75 /// than NewSize, split it into two pieces (where the former is NewSize
76 /// bytes, including the header), and add the new block to the free list.
77 FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList,
81 /// FreeRangeHeader - For a memory block that isn't already allocated, this
82 /// keeps track of the current block and has a pointer to the next free block.
83 /// Free blocks are kept on a circularly linked list.
84 struct FreeRangeHeader : public MemoryRangeHeader {
85 FreeRangeHeader *Prev;
86 FreeRangeHeader *Next;
88 /// getMinBlockSize - Get the minimum size for a memory block. Blocks
89 /// smaller than this size cannot be created.
90 static unsigned getMinBlockSize() {
91 return sizeof(FreeRangeHeader)+sizeof(intptr_t);
94 /// SetEndOfBlockSizeMarker - The word at the end of every free block is
95 /// known to be the size of the free block. Set it for this block.
96 void SetEndOfBlockSizeMarker() {
97 void *EndOfBlock = (char*)this + BlockSize;
98 ((intptr_t *)EndOfBlock)[-1] = BlockSize;
101 FreeRangeHeader *RemoveFromFreeList() {
102 assert(Next->Prev == this && Prev->Next == this && "Freelist broken!");
104 return Prev->Next = Next;
107 void AddToFreeList(FreeRangeHeader *FreeList) {
109 Prev = FreeList->Prev;
114 /// GrowBlock - The block after this block just got deallocated. Merge it
115 /// into the current block.
116 void GrowBlock(uintptr_t NewSize);
118 /// AllocateBlock - Mark this entire block allocated, updating freelists
119 /// etc. This returns a pointer to the circular free-list.
120 FreeRangeHeader *AllocateBlock();
125 /// AllocateBlock - Mark this entire block allocated, updating freelists
126 /// etc. This returns a pointer to the circular free-list.
127 FreeRangeHeader *FreeRangeHeader::AllocateBlock() {
128 assert(!ThisAllocated && !getBlockAfter().PrevAllocated &&
129 "Cannot allocate an allocated block!");
130 // Mark this block allocated.
132 getBlockAfter().PrevAllocated = 1;
134 // Remove it from the free list.
135 return RemoveFromFreeList();
138 /// FreeBlock - Turn an allocated block into a free block, adjusting
139 /// bits in the object headers, and adding an end of region memory block.
140 /// If possible, coalesce this block with neighboring blocks. Return the
141 /// FreeRangeHeader to allocate from.
142 FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) {
143 MemoryRangeHeader *FollowingBlock = &getBlockAfter();
144 assert(ThisAllocated && "This block is already allocated!");
145 assert(FollowingBlock->PrevAllocated && "Flags out of sync!");
147 FreeRangeHeader *FreeListToReturn = FreeList;
149 // If the block after this one is free, merge it into this block.
150 if (!FollowingBlock->ThisAllocated) {
151 FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock;
152 // "FreeList" always needs to be a valid free block. If we're about to
153 // coalesce with it, update our notion of what the free list is.
154 if (&FollowingFreeBlock == FreeList) {
155 FreeList = FollowingFreeBlock.Next;
156 FreeListToReturn = 0;
157 assert(&FollowingFreeBlock != FreeList && "No tombstone block?");
159 FollowingFreeBlock.RemoveFromFreeList();
161 // Include the following block into this one.
162 BlockSize += FollowingFreeBlock.BlockSize;
163 FollowingBlock = &FollowingFreeBlock.getBlockAfter();
165 // Tell the block after the block we are coalescing that this block is
167 FollowingBlock->PrevAllocated = 1;
170 assert(FollowingBlock->ThisAllocated && "Missed coalescing?");
172 if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) {
173 PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize);
174 return FreeListToReturn ? FreeListToReturn : PrevFreeBlock;
177 // Otherwise, mark this block free.
178 FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this;
179 FollowingBlock->PrevAllocated = 0;
180 FreeBlock.ThisAllocated = 0;
182 // Link this into the linked list of free blocks.
183 FreeBlock.AddToFreeList(FreeList);
185 // Add a marker at the end of the block, indicating the size of this free
187 FreeBlock.SetEndOfBlockSizeMarker();
188 return FreeListToReturn ? FreeListToReturn : &FreeBlock;
191 /// GrowBlock - The block after this block just got deallocated. Merge it
192 /// into the current block.
193 void FreeRangeHeader::GrowBlock(uintptr_t NewSize) {
194 assert(NewSize > BlockSize && "Not growing block?");
196 SetEndOfBlockSizeMarker();
197 getBlockAfter().PrevAllocated = 0;
200 /// TrimAllocationToSize - If this allocated block is significantly larger
201 /// than NewSize, split it into two pieces (where the former is NewSize
202 /// bytes, including the header), and add the new block to the free list.
203 FreeRangeHeader *MemoryRangeHeader::
204 TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) {
205 assert(ThisAllocated && getBlockAfter().PrevAllocated &&
206 "Cannot deallocate part of an allocated block!");
208 // Don't allow blocks to be trimmed below minimum required size
209 NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize);
211 // Round up size for alignment of header.
212 unsigned HeaderAlign = __alignof(FreeRangeHeader);
213 NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1);
215 // Size is now the size of the block we will remove from the start of the
217 assert(NewSize <= BlockSize &&
218 "Allocating more space from this block than exists!");
220 // If splitting this block will cause the remainder to be too small, do not
222 if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize())
225 // Otherwise, we splice the required number of bytes out of this block, form
226 // a new block immediately after it, then mark this block allocated.
227 MemoryRangeHeader &FormerNextBlock = getBlockAfter();
229 // Change the size of this block.
232 // Get the new block we just sliced out and turn it into a free block.
233 FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter();
234 NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock;
235 NewNextBlock.ThisAllocated = 0;
236 NewNextBlock.PrevAllocated = 1;
237 NewNextBlock.SetEndOfBlockSizeMarker();
238 FormerNextBlock.PrevAllocated = 0;
239 NewNextBlock.AddToFreeList(FreeList);
240 return &NewNextBlock;
243 //===----------------------------------------------------------------------===//
244 // Memory Block Implementation.
245 //===----------------------------------------------------------------------===//
248 /// DefaultJITMemoryManager - Manage memory for the JIT code generation.
249 /// This splits a large block of MAP_NORESERVE'd memory into two
250 /// sections, one for function stubs, one for the functions themselves. We
251 /// have to do this because we may need to emit a function stub while in the
252 /// middle of emitting a function, and we don't know how large the function we
254 class VISIBILITY_HIDDEN DefaultJITMemoryManager : public JITMemoryManager {
255 bool PoisonMemory; // Whether to poison freed memory.
257 std::vector<sys::MemoryBlock> Blocks; // Memory blocks allocated by the JIT
258 FreeRangeHeader *FreeMemoryList; // Circular list of free blocks.
260 // When emitting code into a memory block, this is the block.
261 MemoryRangeHeader *CurBlock;
263 uint8_t *CurStubPtr, *StubBase;
264 uint8_t *CurGlobalPtr, *GlobalEnd;
265 uint8_t *GOTBase; // Target Specific reserved memory
266 void *DlsymTable; // Stub external symbol information
268 // Centralize memory block allocation.
269 sys::MemoryBlock getNewMemoryBlock(unsigned size);
271 std::map<const Function*, MemoryRangeHeader*> FunctionBlocks;
272 std::map<const Function*, MemoryRangeHeader*> TableBlocks;
274 DefaultJITMemoryManager();
275 ~DefaultJITMemoryManager();
278 void SetDlsymTable(void *);
280 uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize,
283 /// startFunctionBody - When a function starts, allocate a block of free
284 /// executable memory, returning a pointer to it and its actual size.
285 uint8_t *startFunctionBody(const Function *F, uintptr_t &ActualSize) {
287 FreeRangeHeader* candidateBlock = FreeMemoryList;
288 FreeRangeHeader* head = FreeMemoryList;
289 FreeRangeHeader* iter = head->Next;
291 uintptr_t largest = candidateBlock->BlockSize;
293 // Search for the largest free block
294 while (iter != head) {
295 if (iter->BlockSize > largest) {
296 largest = iter->BlockSize;
297 candidateBlock = iter;
302 // Select this candidate block for allocation
303 CurBlock = candidateBlock;
305 // Allocate the entire memory block.
306 FreeMemoryList = candidateBlock->AllocateBlock();
307 ActualSize = CurBlock->BlockSize-sizeof(MemoryRangeHeader);
308 return (uint8_t *)(CurBlock+1);
311 /// endFunctionBody - The function F is now allocated, and takes the memory
312 /// in the range [FunctionStart,FunctionEnd).
313 void endFunctionBody(const Function *F, uint8_t *FunctionStart,
314 uint8_t *FunctionEnd) {
315 assert(FunctionEnd > FunctionStart);
316 assert(FunctionStart == (uint8_t *)(CurBlock+1) &&
317 "Mismatched function start/end!");
319 uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock;
320 FunctionBlocks[F] = CurBlock;
322 // Release the memory at the end of this block that isn't needed.
323 FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
326 /// allocateSpace - Allocate a memory block of the given size.
327 uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) {
328 CurBlock = FreeMemoryList;
329 FreeMemoryList = FreeMemoryList->AllocateBlock();
331 uint8_t *result = (uint8_t *)(CurBlock + 1);
333 if (Alignment == 0) Alignment = 1;
334 result = (uint8_t*)(((intptr_t)result+Alignment-1) &
335 ~(intptr_t)(Alignment-1));
337 uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock;
338 FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
343 /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace,
344 /// this method does not touch the current block and can be called at any
346 uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) {
347 uint8_t *Result = CurGlobalPtr;
349 // Align the pointer.
350 if (Alignment == 0) Alignment = 1;
351 Result = (uint8_t*)(((uintptr_t)Result + Alignment-1) &
352 ~(uintptr_t)(Alignment-1));
354 // Move the current global pointer forward.
355 CurGlobalPtr += Result - CurGlobalPtr + Size;
357 // Check for overflow.
358 if (CurGlobalPtr > GlobalEnd) {
359 // FIXME: Allocate more memory.
360 llvm_report_error("JIT ran out of memory for globals!");
366 /// startExceptionTable - Use startFunctionBody to allocate memory for the
367 /// function's exception table.
368 uint8_t* startExceptionTable(const Function* F, uintptr_t &ActualSize) {
369 return startFunctionBody(F, ActualSize);
372 /// endExceptionTable - The exception table of F is now allocated,
373 /// and takes the memory in the range [TableStart,TableEnd).
374 void endExceptionTable(const Function *F, uint8_t *TableStart,
375 uint8_t *TableEnd, uint8_t* FrameRegister) {
376 assert(TableEnd > TableStart);
377 assert(TableStart == (uint8_t *)(CurBlock+1) &&
378 "Mismatched table start/end!");
380 uintptr_t BlockSize = TableEnd - (uint8_t *)CurBlock;
381 TableBlocks[F] = CurBlock;
383 // Release the memory at the end of this block that isn't needed.
384 FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
387 uint8_t *getGOTBase() const {
391 void *getDlsymTable() const {
395 /// deallocateMemForFunction - Deallocate all memory for the specified
397 void deallocateMemForFunction(const Function *F) {
398 std::map<const Function*, MemoryRangeHeader*>::iterator
399 I = FunctionBlocks.find(F);
400 if (I == FunctionBlocks.end()) return;
402 // Find the block that is allocated for this function.
403 MemoryRangeHeader *MemRange = I->second;
404 assert(MemRange->ThisAllocated && "Block isn't allocated!");
406 // Fill the buffer with garbage!
408 memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
412 FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
414 // Finally, remove this entry from FunctionBlocks.
415 FunctionBlocks.erase(I);
417 I = TableBlocks.find(F);
418 if (I == TableBlocks.end()) return;
420 // Find the block that is allocated for this function.
421 MemRange = I->second;
422 assert(MemRange->ThisAllocated && "Block isn't allocated!");
424 // Fill the buffer with garbage!
426 memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
430 FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
432 // Finally, remove this entry from TableBlocks.
433 TableBlocks.erase(I);
436 /// setMemoryWritable - When code generation is in progress,
437 /// the code pages may need permissions changed.
438 void setMemoryWritable(void)
440 for (unsigned i = 0, e = Blocks.size(); i != e; ++i)
441 sys::Memory::setWritable(Blocks[i]);
443 /// setMemoryExecutable - When code generation is done and we're ready to
444 /// start execution, the code pages may need permissions changed.
445 void setMemoryExecutable(void)
447 for (unsigned i = 0, e = Blocks.size(); i != e; ++i)
448 sys::Memory::setExecutable(Blocks[i]);
451 /// setPoisonMemory - Controls whether we write garbage over freed memory.
453 void setPoisonMemory(bool poison) {
454 PoisonMemory = poison;
459 DefaultJITMemoryManager::DefaultJITMemoryManager() {
463 PoisonMemory = false;
466 // Allocate a 16M block of memory for functions.
467 #if defined(__APPLE__) && defined(__arm__)
468 sys::MemoryBlock MemBlock = getNewMemoryBlock(4 << 20);
470 sys::MemoryBlock MemBlock = getNewMemoryBlock(16 << 20);
473 uint8_t *MemBase = static_cast<uint8_t*>(MemBlock.base());
475 // Allocate stubs backwards to the base, globals forward from the stubs, and
476 // functions forward after globals.
478 CurStubPtr = MemBase + 512*1024; // Use 512k for stubs, working backwards.
479 CurGlobalPtr = CurStubPtr; // Use 2M for globals, working forwards.
480 GlobalEnd = CurGlobalPtr + 2*1024*1024;
482 // We set up the memory chunk with 4 mem regions, like this:
484 // [ Free #0 ] -> Large space to allocate functions from.
485 // [ Allocated #1 ] -> Tiny space to separate regions.
486 // [ Free #2 ] -> Tiny space so there is always at least 1 free block.
487 // [ Allocated #3 ] -> Tiny space to prevent looking past end of block.
490 // The last three blocks are never deallocated or touched.
492 // Add MemoryRangeHeader to the end of the memory region, indicating that
493 // the space after the block of memory is allocated. This is block #3.
494 MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1;
495 Mem3->ThisAllocated = 1;
496 Mem3->PrevAllocated = 0;
499 /// Add a tiny free region so that the free list always has one entry.
500 FreeRangeHeader *Mem2 =
501 (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize());
502 Mem2->ThisAllocated = 0;
503 Mem2->PrevAllocated = 1;
504 Mem2->BlockSize = FreeRangeHeader::getMinBlockSize();
505 Mem2->SetEndOfBlockSizeMarker();
506 Mem2->Prev = Mem2; // Mem2 *is* the free list for now.
509 /// Add a tiny allocated region so that Mem2 is never coalesced away.
510 MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1;
511 Mem1->ThisAllocated = 1;
512 Mem1->PrevAllocated = 0;
513 Mem1->BlockSize = (char*)Mem2 - (char*)Mem1;
515 // Add a FreeRangeHeader to the start of the function body region, indicating
516 // that the space is free. Mark the previous block allocated so we never look
518 FreeRangeHeader *Mem0 = (FreeRangeHeader*)GlobalEnd;
519 Mem0->ThisAllocated = 0;
520 Mem0->PrevAllocated = 1;
521 Mem0->BlockSize = (char*)Mem1-(char*)Mem0;
522 Mem0->SetEndOfBlockSizeMarker();
523 Mem0->AddToFreeList(Mem2);
525 // Start out with the freelist pointing to Mem0.
526 FreeMemoryList = Mem0;
532 void DefaultJITMemoryManager::AllocateGOT() {
533 assert(GOTBase == 0 && "Cannot allocate the got multiple times");
534 GOTBase = new uint8_t[sizeof(void*) * 8192];
538 void DefaultJITMemoryManager::SetDlsymTable(void *ptr) {
542 DefaultJITMemoryManager::~DefaultJITMemoryManager() {
543 for (unsigned i = 0, e = Blocks.size(); i != e; ++i)
544 sys::Memory::ReleaseRWX(Blocks[i]);
550 uint8_t *DefaultJITMemoryManager::allocateStub(const GlobalValue* F,
552 unsigned Alignment) {
553 CurStubPtr -= StubSize;
554 CurStubPtr = (uint8_t*)(((intptr_t)CurStubPtr) &
555 ~(intptr_t)(Alignment-1));
556 if (CurStubPtr < StubBase) {
557 // FIXME: allocate a new block
558 llvm_report_error("JIT ran out of memory for function stubs!");
563 sys::MemoryBlock DefaultJITMemoryManager::getNewMemoryBlock(unsigned size) {
564 // Allocate a new block close to the last one.
565 const sys::MemoryBlock *BOld = Blocks.empty() ? 0 : &Blocks.back();
567 sys::MemoryBlock B = sys::Memory::AllocateRWX(size, BOld, &ErrMsg);
569 llvm_report_error("Allocation failed when allocating new memory in the"
577 JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() {
578 return new DefaultJITMemoryManager();