[SectionMemoryManager] Make better use of virtual memory

Summary: On Windows, the allocation granularity can be significantly
larger than a page (64K), so with many small objects, just clearing
the FreeMem list rapidly leaks quite a bit of virtual memory space
(if not rss). Fix that by only removing those parts of the FreeMem
blocks that overlap pages for which we are applying memory permissions,
rather than dropping the FreeMem blocks entirely.

Reviewers: lhames

Subscribers: llvm-commits

Differential Revision: http://reviews.llvm.org/D15202

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@255760 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/llvm/ExecutionEngine/SectionMemoryManager.h b/include/llvm/ExecutionEngine/SectionMemoryManager.h
index 6135bc68e9a68cc0782432b460df63da63feecac..7bb96eb8b71b20b1b7b942f487095e76073e106d 100644 (file)
--- a/include/llvm/ExecutionEngine/SectionMemoryManager.h
+++ b/include/llvm/ExecutionEngine/SectionMemoryManager.h
@@ -83,25 +83,28 @@ public:
   virtual void invalidateInstructionCache();
 
 private:
   virtual void invalidateInstructionCache();
 
 private:

+  struct FreeMemBlock {
+    // The actual block of free memory
+    sys::MemoryBlock Free;
+    // If there is a pending allocation from the same reservation right before
+    // this block, store it's index in PendingMem, to be able to update the
+    // pending region if part of this block is allocated, rather than having to
+    // create a new one
+    unsigned PendingPrefixIndex;
+  };
+

   struct MemoryGroup {
   struct MemoryGroup {

-      // PendingMem contains all allocated memory blocks
-      // which have not yet had their permissions set. Note
-      // that this tracks memory blocks that have been given to
-      // this memory manager by the system, not those
-      // given out to the user. In particular, the memory manager
-      // will give out subblocks of these MemoryBlocks in response
-      // to user requests. We track which subblocks have not beeen
-      // given out yet in `FreeMem`.
-      SmallVector<sys::MemoryBlock, 16> PendingMem;
-      SmallVector<sys::MemoryBlock, 16> FreeMem;
-
-      // All allocated memory blocks that have had their permissions
-      // set (i.e. that have been finalized). Because of this, we may
-      // not give out subblocks of this memory to the user anymore,
-      // even if those subblocks have not been previously given out.
-      SmallVector<sys::MemoryBlock, 16> AllocatedMem;
-
-      sys::MemoryBlock Near;
+    // PendingMem contains all blocks of memory (subblocks of AllocatedMem)
+    // which have not yet had their permissions applied, but have been given
+    // out to the user. FreeMem contains all block of memory, which have
+    // neither had their permissions applied, nor been given out to the user.
+    SmallVector<sys::MemoryBlock, 16> PendingMem;
+    SmallVector<FreeMemBlock, 16> FreeMem;
+
+    // All memory blocks that have been requested from the system
+    SmallVector<sys::MemoryBlock, 16> AllocatedMem;
+
+    sys::MemoryBlock Near;

   };
 
   uint8_t *allocateSection(MemoryGroup &MemGroup, uintptr_t Size,
   };
 
   uint8_t *allocateSection(MemoryGroup &MemGroup, uintptr_t Size,


@@ -118,4 +121,3 @@ private:
 }
 
 #endif // LLVM_EXECUTION_ENGINE_SECTION_MEMORY_MANAGER_H
 }
 
 #endif // LLVM_EXECUTION_ENGINE_SECTION_MEMORY_MANAGER_H

-

diff --git a/lib/ExecutionEngine/SectionMemoryManager.cpp b/lib/ExecutionEngine/SectionMemoryManager.cpp
index 5e89a945b80d109fa7f2bc30829d3d674fd0cb80..e2f220862cf75812e03b83750c50ce74542de970 100644 (file)
--- a/lib/ExecutionEngine/SectionMemoryManager.cpp
+++ b/lib/ExecutionEngine/SectionMemoryManager.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Config/config.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Config/config.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/Support/MathExtras.h"

+#include "llvm/Support/Process.h"

 
 namespace llvm {
 
 
 namespace llvm {
 


@@ -48,14 +49,27 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup,
 
   // Look in the list of free memory regions and use a block there if one
   // is available.
 
   // Look in the list of free memory regions and use a block there if one
   // is available.

-  for (sys::MemoryBlock &MB : MemGroup.FreeMem) {
-    if (MB.size() >= RequiredSize) {
-      Addr = (uintptr_t)MB.base();
-      uintptr_t EndOfBlock = Addr + MB.size();
+  for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
+    if (FreeMB.Free.size() >= RequiredSize) {
+      Addr = (uintptr_t)FreeMB.Free.base();
+      uintptr_t EndOfBlock = Addr + FreeMB.Free.size();

       // Align the address.
       Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
       // Align the address.
       Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);

-      // Store cutted free memory block.
-      MB = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
+
+      if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
+        // The part of the block we're giving out to the user is now pending
+        MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
+
+        // Remember this pending block, such that future allocations can just
+        // modify it rather than creating a new one
+        FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
+      } else {
+        sys::MemoryBlock &PendingMB = MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
+        PendingMB = sys::MemoryBlock(PendingMB.base(), Addr + Size - (uintptr_t)PendingMB.base());
+      }
+
+      // Remember how much free space is now left in this block
+      FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);

       return (uint8_t*)Addr;
     }
   }
       return (uint8_t*)Addr;
     }
   }


@@ -83,18 +97,26 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup,
   // Save this address as the basis for our next request
   MemGroup.Near = MB;
 
   // Save this address as the basis for our next request
   MemGroup.Near = MB;
 

-  MemGroup.PendingMem.push_back(MB);
+  // Remember that we allocated this memory
+  MemGroup.AllocatedMem.push_back(MB);

   Addr = (uintptr_t)MB.base();
   uintptr_t EndOfBlock = Addr + MB.size();
 
   // Align the address.
   Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
 
   Addr = (uintptr_t)MB.base();
   uintptr_t EndOfBlock = Addr + MB.size();
 
   // Align the address.
   Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
 

+  // The part of the block we're giving out to the user is now pending
+  MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
+

   // The allocateMappedMemory may allocate much more memory than we need. In
   // this case, we store the unused memory as a free memory block.
   unsigned FreeSize = EndOfBlock-Addr-Size;
   // The allocateMappedMemory may allocate much more memory than we need. In
   // this case, we store the unused memory as a free memory block.
   unsigned FreeSize = EndOfBlock-Addr-Size;

-  if (FreeSize > 16)
-    MemGroup.FreeMem.push_back(sys::MemoryBlock((void*)(Addr + Size), FreeSize));
+  if (FreeSize > 16) {
+    FreeMemBlock FreeMB;
+    FreeMB.Free = sys::MemoryBlock((void*)(Addr + Size), FreeSize);
+    FreeMB.PendingPrefixIndex = (unsigned)-1;
+    MemGroup.FreeMem.push_back(FreeMB);
+  }

 
   // Return aligned address
   return (uint8_t*)Addr;
 
   // Return aligned address
   return (uint8_t*)Addr;


@@ -105,9 +127,6 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg)
   // FIXME: Should in-progress permissions be reverted if an error occurs?
   std::error_code ec;
 
   // FIXME: Should in-progress permissions be reverted if an error occurs?
   std::error_code ec;
 

-  // Don't allow free memory blocks to be used after setting protection flags.
-  CodeMem.FreeMem.clear();
-

   // Make code memory executable.
   ec = applyMemoryGroupPermissions(CodeMem,
                                    sys::Memory::MF_READ | sys::Memory::MF_EXEC);
   // Make code memory executable.
   ec = applyMemoryGroupPermissions(CodeMem,
                                    sys::Memory::MF_READ | sys::Memory::MF_EXEC);


@@ -138,25 +157,52 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg)
   // relocations) will get to the data cache but not to the instruction cache.
   invalidateInstructionCache();
 
   // relocations) will get to the data cache but not to the instruction cache.
   invalidateInstructionCache();
 

-  // Now, remember that we have successfully applied the permissions to avoid
-  // having to apply them again.
-  CodeMem.AllocatedMem.append(CodeMem.PendingMem.begin(),CodeMem.PendingMem.end());
-  CodeMem.PendingMem.clear();
+  return false;
+}

 
 

-  RODataMem.AllocatedMem.append(RODataMem.PendingMem.begin(),RODataMem.PendingMem.end());
-  RODataMem.PendingMem.clear();
+static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) {
+  static const size_t PageSize = sys::Process::getPageSize();

 
 

-  return false;
+  size_t StartOverlap =
+      (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize;
+
+  size_t TrimmedSize = M.size();
+  TrimmedSize -= StartOverlap;
+  TrimmedSize -= TrimmedSize % PageSize;
+
+  sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), TrimmedSize);
+
+  assert(((uintptr_t)Trimmed.base() % PageSize) == 0);
+  assert((Trimmed.size() % PageSize) == 0);
+  assert(M.base() <= Trimmed.base() && Trimmed.size() <= M.size());
+
+  return Trimmed;

 }
 
 }
 

+

 std::error_code
 SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
                                                   unsigned Permissions) {
 std::error_code
 SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
                                                   unsigned Permissions) {

-

   for (sys::MemoryBlock &MB : MemGroup.PendingMem)
     if (std::error_code EC = sys::Memory::protectMappedMemory(MB, Permissions))
       return EC;
 
   for (sys::MemoryBlock &MB : MemGroup.PendingMem)
     if (std::error_code EC = sys::Memory::protectMappedMemory(MB, Permissions))
       return EC;
 

+  MemGroup.PendingMem.clear();
+
+  // Now go through free blocks and trim any of them that don't span the entire
+  // page because one of the pending blocks may have overlapped it.
+  for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
+    FreeMB.Free = trimBlockToPageSize(FreeMB.Free);
+    // We cleared the PendingMem list, so all these pointers are now invalid
+    FreeMB.PendingPrefixIndex = (unsigned)-1;
+  }
+
+  // Remove all blocks which are now empty
+  MemGroup.FreeMem.erase(
+      std::remove_if(MemGroup.FreeMem.begin(), MemGroup.FreeMem.end(),
+                     [](FreeMemBlock &FreeMB) { return FreeMB.Free.size() == 0; }),
+      MemGroup.FreeMem.end());
+

   return std::error_code();
 }
 
   return std::error_code();
 }
 


@@ -169,10 +215,7 @@ SectionMemoryManager::~SectionMemoryManager() {
   for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {
     for (sys::MemoryBlock &Block : Group->AllocatedMem)
       sys::Memory::releaseMappedMemory(Block);
   for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {
     for (sys::MemoryBlock &Block : Group->AllocatedMem)
       sys::Memory::releaseMappedMemory(Block);

-    for (sys::MemoryBlock &Block : Group->PendingMem)
-      sys::Memory::releaseMappedMemory(Block);

   }
 }
 
 } // namespace llvm
   }
 }
 
 } // namespace llvm

-

diff --git a/lib/Support/Unix/Memory.inc b/lib/Support/Unix/Memory.inc
index 7bffdf38773a5f33ceb6522da7b9bd4415b148ea..d70319168b84dfd90a92ba30be4f6579ba285279 100644 (file)
--- a/lib/Support/Unix/Memory.inc
+++ b/lib/Support/Unix/Memory.inc
@@ -152,6 +152,7 @@ Memory::releaseMappedMemory(MemoryBlock &M) {
 
 std::error_code
 Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) {
 
 std::error_code
 Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) {

+  static const size_t PageSize = Process::getPageSize();

   if (M.Address == nullptr || M.Size == 0)
     return std::error_code();
 
   if (M.Address == nullptr || M.Size == 0)
     return std::error_code();
 


@@ -160,7 +161,7 @@ Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) {
 
   int Protect = getPosixProtectionFlags(Flags);
 
 
   int Protect = getPosixProtectionFlags(Flags);
 

-  int Result = ::mprotect(M.Address, M.Size, Protect);
+  int Result = ::mprotect((void*)((uintptr_t)M.Address & ~(PageSize-1)), PageSize*((M.Size+PageSize-1)/PageSize), Protect);

   if (Result != 0)
     return std::error_code(errno, std::generic_category());
 
   if (Result != 0)
     return std::error_code(errno, std::generic_category());
 


@@ -180,7 +181,7 @@ Memory::AllocateRWX(size_t NumBytes, const MemoryBlock* NearBlock,
                     std::string *ErrMsg) {
   if (NumBytes == 0) return MemoryBlock();
 
                     std::string *ErrMsg) {
   if (NumBytes == 0) return MemoryBlock();
 

-  size_t PageSize = Process::getPageSize();
+  static const size_t PageSize = Process::getPageSize();

   size_t NumPages = (NumBytes+PageSize-1)/PageSize;
 
   int fd = -1;
   size_t NumPages = (NumBytes+PageSize-1)/PageSize;
 
   int fd = -1;