[llvm] r255760 - [SectionMemoryManager] Make better use of virtual memory
Keno Fischer via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 16 03:13:23 PST 2015
Author: kfischer
Date: Wed Dec 16 05:13:23 2015
New Revision: 255760
URL: http://llvm.org/viewvc/llvm-project?rev=255760&view=rev
Log:
[SectionMemoryManager] Make better use of virtual memory
Summary: On Windows, the allocation granularity can be significantly
larger than a page (64K), so with many small objects, just clearing
the FreeMem list rapidly leaks quite a bit of virtual memory space
(if not rss). Fix that by only removing those parts of the FreeMem
blocks that overlap pages for which we are applying memory permissions,
rather than dropping the FreeMem blocks entirely.
Reviewers: lhames
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D15202
Modified:
llvm/trunk/include/llvm/ExecutionEngine/SectionMemoryManager.h
llvm/trunk/lib/ExecutionEngine/SectionMemoryManager.cpp
llvm/trunk/lib/Support/Unix/Memory.inc
Modified: llvm/trunk/include/llvm/ExecutionEngine/SectionMemoryManager.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/ExecutionEngine/SectionMemoryManager.h?rev=255760&r1=255759&r2=255760&view=diff
==============================================================================
--- llvm/trunk/include/llvm/ExecutionEngine/SectionMemoryManager.h (original)
+++ llvm/trunk/include/llvm/ExecutionEngine/SectionMemoryManager.h Wed Dec 16 05:13:23 2015
@@ -83,25 +83,28 @@ public:
virtual void invalidateInstructionCache();
private:
+ struct FreeMemBlock {
+ // The actual block of free memory
+ sys::MemoryBlock Free;
+ // If there is a pending allocation from the same reservation right before
+ // this block, store it's index in PendingMem, to be able to update the
+ // pending region if part of this block is allocated, rather than having to
+ // create a new one
+ unsigned PendingPrefixIndex;
+ };
+
struct MemoryGroup {
- // PendingMem contains all allocated memory blocks
- // which have not yet had their permissions set. Note
- // that this tracks memory blocks that have been given to
- // this memory manager by the system, not those
- // given out to the user. In particular, the memory manager
- // will give out subblocks of these MemoryBlocks in response
- // to user requests. We track which subblocks have not beeen
- // given out yet in `FreeMem`.
- SmallVector<sys::MemoryBlock, 16> PendingMem;
- SmallVector<sys::MemoryBlock, 16> FreeMem;
-
- // All allocated memory blocks that have had their permissions
- // set (i.e. that have been finalized). Because of this, we may
- // not give out subblocks of this memory to the user anymore,
- // even if those subblocks have not been previously given out.
- SmallVector<sys::MemoryBlock, 16> AllocatedMem;
+ // PendingMem contains all blocks of memory (subblocks of AllocatedMem)
+ // which have not yet had their permissions applied, but have been given
+ // out to the user. FreeMem contains all block of memory, which have
+ // neither had their permissions applied, nor been given out to the user.
+ SmallVector<sys::MemoryBlock, 16> PendingMem;
+ SmallVector<FreeMemBlock, 16> FreeMem;
- sys::MemoryBlock Near;
+ // All memory blocks that have been requested from the system
+ SmallVector<sys::MemoryBlock, 16> AllocatedMem;
+
+ sys::MemoryBlock Near;
};
uint8_t *allocateSection(MemoryGroup &MemGroup, uintptr_t Size,
@@ -118,4 +121,3 @@ private:
}
#endif // LLVM_EXECUTION_ENGINE_SECTION_MEMORY_MANAGER_H
-
Modified: llvm/trunk/lib/ExecutionEngine/SectionMemoryManager.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/ExecutionEngine/SectionMemoryManager.cpp?rev=255760&r1=255759&r2=255760&view=diff
==============================================================================
--- llvm/trunk/lib/ExecutionEngine/SectionMemoryManager.cpp (original)
+++ llvm/trunk/lib/ExecutionEngine/SectionMemoryManager.cpp Wed Dec 16 05:13:23 2015
@@ -15,6 +15,7 @@
#include "llvm/Config/config.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/Process.h"
namespace llvm {
@@ -48,14 +49,27 @@ uint8_t *SectionMemoryManager::allocateS
// Look in the list of free memory regions and use a block there if one
// is available.
- for (sys::MemoryBlock &MB : MemGroup.FreeMem) {
- if (MB.size() >= RequiredSize) {
- Addr = (uintptr_t)MB.base();
- uintptr_t EndOfBlock = Addr + MB.size();
+ for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
+ if (FreeMB.Free.size() >= RequiredSize) {
+ Addr = (uintptr_t)FreeMB.Free.base();
+ uintptr_t EndOfBlock = Addr + FreeMB.Free.size();
// Align the address.
Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
- // Store cutted free memory block.
- MB = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
+
+ if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
+ // The part of the block we're giving out to the user is now pending
+ MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
+
+ // Remember this pending block, such that future allocations can just
+ // modify it rather than creating a new one
+ FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
+ } else {
+ sys::MemoryBlock &PendingMB = MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
+ PendingMB = sys::MemoryBlock(PendingMB.base(), Addr + Size - (uintptr_t)PendingMB.base());
+ }
+
+ // Remember how much free space is now left in this block
+ FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
return (uint8_t*)Addr;
}
}
@@ -83,18 +97,26 @@ uint8_t *SectionMemoryManager::allocateS
// Save this address as the basis for our next request
MemGroup.Near = MB;
- MemGroup.PendingMem.push_back(MB);
+ // Remember that we allocated this memory
+ MemGroup.AllocatedMem.push_back(MB);
Addr = (uintptr_t)MB.base();
uintptr_t EndOfBlock = Addr + MB.size();
// Align the address.
Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
+ // The part of the block we're giving out to the user is now pending
+ MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
+
// The allocateMappedMemory may allocate much more memory than we need. In
// this case, we store the unused memory as a free memory block.
unsigned FreeSize = EndOfBlock-Addr-Size;
- if (FreeSize > 16)
- MemGroup.FreeMem.push_back(sys::MemoryBlock((void*)(Addr + Size), FreeSize));
+ if (FreeSize > 16) {
+ FreeMemBlock FreeMB;
+ FreeMB.Free = sys::MemoryBlock((void*)(Addr + Size), FreeSize);
+ FreeMB.PendingPrefixIndex = (unsigned)-1;
+ MemGroup.FreeMem.push_back(FreeMB);
+ }
// Return aligned address
return (uint8_t*)Addr;
@@ -105,9 +127,6 @@ bool SectionMemoryManager::finalizeMemor
// FIXME: Should in-progress permissions be reverted if an error occurs?
std::error_code ec;
- // Don't allow free memory blocks to be used after setting protection flags.
- CodeMem.FreeMem.clear();
-
// Make code memory executable.
ec = applyMemoryGroupPermissions(CodeMem,
sys::Memory::MF_READ | sys::Memory::MF_EXEC);
@@ -138,25 +157,52 @@ bool SectionMemoryManager::finalizeMemor
// relocations) will get to the data cache but not to the instruction cache.
invalidateInstructionCache();
- // Now, remember that we have successfully applied the permissions to avoid
- // having to apply them again.
- CodeMem.AllocatedMem.append(CodeMem.PendingMem.begin(),CodeMem.PendingMem.end());
- CodeMem.PendingMem.clear();
+ return false;
+}
- RODataMem.AllocatedMem.append(RODataMem.PendingMem.begin(),RODataMem.PendingMem.end());
- RODataMem.PendingMem.clear();
+static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) {
+ static const size_t PageSize = sys::Process::getPageSize();
- return false;
+ size_t StartOverlap =
+ (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize;
+
+ size_t TrimmedSize = M.size();
+ TrimmedSize -= StartOverlap;
+ TrimmedSize -= TrimmedSize % PageSize;
+
+ sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), TrimmedSize);
+
+ assert(((uintptr_t)Trimmed.base() % PageSize) == 0);
+ assert((Trimmed.size() % PageSize) == 0);
+ assert(M.base() <= Trimmed.base() && Trimmed.size() <= M.size());
+
+ return Trimmed;
}
+
std::error_code
SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
unsigned Permissions) {
-
for (sys::MemoryBlock &MB : MemGroup.PendingMem)
if (std::error_code EC = sys::Memory::protectMappedMemory(MB, Permissions))
return EC;
+ MemGroup.PendingMem.clear();
+
+ // Now go through free blocks and trim any of them that don't span the entire
+ // page because one of the pending blocks may have overlapped it.
+ for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
+ FreeMB.Free = trimBlockToPageSize(FreeMB.Free);
+ // We cleared the PendingMem list, so all these pointers are now invalid
+ FreeMB.PendingPrefixIndex = (unsigned)-1;
+ }
+
+ // Remove all blocks which are now empty
+ MemGroup.FreeMem.erase(
+ std::remove_if(MemGroup.FreeMem.begin(), MemGroup.FreeMem.end(),
+ [](FreeMemBlock &FreeMB) { return FreeMB.Free.size() == 0; }),
+ MemGroup.FreeMem.end());
+
return std::error_code();
}
@@ -169,10 +215,7 @@ SectionMemoryManager::~SectionMemoryMana
for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {
for (sys::MemoryBlock &Block : Group->AllocatedMem)
sys::Memory::releaseMappedMemory(Block);
- for (sys::MemoryBlock &Block : Group->PendingMem)
- sys::Memory::releaseMappedMemory(Block);
}
}
} // namespace llvm
-
Modified: llvm/trunk/lib/Support/Unix/Memory.inc
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Support/Unix/Memory.inc?rev=255760&r1=255759&r2=255760&view=diff
==============================================================================
--- llvm/trunk/lib/Support/Unix/Memory.inc (original)
+++ llvm/trunk/lib/Support/Unix/Memory.inc Wed Dec 16 05:13:23 2015
@@ -152,6 +152,7 @@ Memory::releaseMappedMemory(MemoryBlock
std::error_code
Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) {
+ static const size_t PageSize = Process::getPageSize();
if (M.Address == nullptr || M.Size == 0)
return std::error_code();
@@ -160,7 +161,7 @@ Memory::protectMappedMemory(const Memory
int Protect = getPosixProtectionFlags(Flags);
- int Result = ::mprotect(M.Address, M.Size, Protect);
+ int Result = ::mprotect((void*)((uintptr_t)M.Address & ~(PageSize-1)), PageSize*((M.Size+PageSize-1)/PageSize), Protect);
if (Result != 0)
return std::error_code(errno, std::generic_category());
@@ -180,7 +181,7 @@ Memory::AllocateRWX(size_t NumBytes, con
std::string *ErrMsg) {
if (NumBytes == 0) return MemoryBlock();
- size_t PageSize = Process::getPageSize();
+ static const size_t PageSize = Process::getPageSize();
size_t NumPages = (NumBytes+PageSize-1)/PageSize;
int fd = -1;
More information about the llvm-commits
mailing list