Fix performance problem in long-running SectionMemoryManagers

Summary:
Without this patch, the memory manager would call `mprotect` on every memory
region it ever allocated whenever it wanted to finalize memory (i.e. not just
the ones it just allocated). This caused terrible performance problems for
long running memory managers. In one particular compile heavy julia benchmark,
we were spending 50% of time in `mprotect` if running under MCJIT.

Fix this by splitting allocated memory blocks into those on which memory
permissions have been set and those on which they haven't and only running
`mprotect` on the latter.

Reviewers: lhames

Subscribers: reames, llvm-commits

Differential Revision: http://reviews.llvm.org/D13156

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@248981 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/llvm/ExecutionEngine/SectionMemoryManager.h b/include/llvm/ExecutionEngine/SectionMemoryManager.h
index 0b0dcb021f148729f4ab953d01e200ab8c4f8e67..6135bc68e9a68cc0782432b460df63da63feecac 100644 (file)
--- a/include/llvm/ExecutionEngine/SectionMemoryManager.h
+++ b/include/llvm/ExecutionEngine/SectionMemoryManager.h
@@ -84,8 +84,23 @@ public:
 
 private:
   struct MemoryGroup {
-      SmallVector<sys::MemoryBlock, 16> AllocatedMem;
+      // PendingMem contains all allocated memory blocks
+      // which have not yet had their permissions set. Note
+      // that this tracks memory blocks that have been given to
+      // this memory manager by the system, not those
+      // given out to the user. In particular, the memory manager
+      // will give out subblocks of these MemoryBlocks in response
+      // to user requests. We track which subblocks have not beeen
+      // given out yet in `FreeMem`.
+      SmallVector<sys::MemoryBlock, 16> PendingMem;
       SmallVector<sys::MemoryBlock, 16> FreeMem;
+
+      // All allocated memory blocks that have had their permissions
+      // set (i.e. that have been finalized). Because of this, we may
+      // not give out subblocks of this memory to the user anymore,
+      // even if those subblocks have not been previously given out.
+      SmallVector<sys::MemoryBlock, 16> AllocatedMem;
+
       sys::MemoryBlock Near;
   };
 

diff --git a/lib/ExecutionEngine/SectionMemoryManager.cpp b/lib/ExecutionEngine/SectionMemoryManager.cpp
index b22c6db8c3716b2e39b1607dbaf9ea8207f33b96..5e89a945b80d109fa7f2bc30829d3d674fd0cb80 100644 (file)
--- a/lib/ExecutionEngine/SectionMemoryManager.cpp
+++ b/lib/ExecutionEngine/SectionMemoryManager.cpp
@@ -83,7 +83,7 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup,
   // Save this address as the basis for our next request
   MemGroup.Near = MB;
 
-  MemGroup.AllocatedMem.push_back(MB);
+  MemGroup.PendingMem.push_back(MB);
   Addr = (uintptr_t)MB.base();
   uintptr_t EndOfBlock = Addr + MB.size();
 
@@ -138,6 +138,14 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg)
   // relocations) will get to the data cache but not to the instruction cache.
   invalidateInstructionCache();
 
+  // Now, remember that we have successfully applied the permissions to avoid
+  // having to apply them again.
+  CodeMem.AllocatedMem.append(CodeMem.PendingMem.begin(),CodeMem.PendingMem.end());
+  CodeMem.PendingMem.clear();
+
+  RODataMem.AllocatedMem.append(RODataMem.PendingMem.begin(),RODataMem.PendingMem.end());
+  RODataMem.PendingMem.clear();
+
   return false;
 }
 
@@ -145,7 +153,7 @@ std::error_code
 SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
                                                   unsigned Permissions) {
 
-  for (sys::MemoryBlock &MB : MemGroup.AllocatedMem)
+  for (sys::MemoryBlock &MB : MemGroup.PendingMem)
     if (std::error_code EC = sys::Memory::protectMappedMemory(MB, Permissions))
       return EC;
 
@@ -153,14 +161,17 @@ SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
 }
 
 void SectionMemoryManager::invalidateInstructionCache() {
-  for (sys::MemoryBlock &Block : CodeMem.AllocatedMem)
+  for (sys::MemoryBlock &Block : CodeMem.PendingMem)
     sys::Memory::InvalidateInstructionCache(Block.base(), Block.size());
 }
 
 SectionMemoryManager::~SectionMemoryManager() {
-  for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem})
+  for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {
     for (sys::MemoryBlock &Block : Group->AllocatedMem)
       sys::Memory::releaseMappedMemory(Block);
+    for (sys::MemoryBlock &Block : Group->PendingMem)
+      sys::Memory::releaseMappedMemory(Block);
+  }
 }
 
 } // namespace llvm