[compiler-rt] [scudo] Don't preserve space for regions at init() (DO NOT MERGE) (PR #74531)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Feb 26 15:02:58 PST 2024
https://github.com/ChiaHungDuan updated https://github.com/llvm/llvm-project/pull/74531
>From c1702fe2cc15d3a6718aa70cf696cb1a6c39cf01 Mon Sep 17 00:00:00 2001
From: Chia-hung Duan <chiahungduan at google.com>
Date: Thu, 26 Oct 2023 00:00:10 +0000
Subject: [PATCH 1/5] [scudo] Store more blocks in each TransferBatch
Instead of always storing the same number of blocks as cached, we prefer
increasing the utilization by saving more blocks in a single
TransferBatch. This may slightly impact the performance, but it will
save a lot of memory used by BatchClassId (especially for larger
blocks).
---
.../lib/scudo/standalone/allocator_common.h | 7 +
compiler-rt/lib/scudo/standalone/primary32.h | 106 +++++++------
compiler-rt/lib/scudo/standalone/primary64.h | 150 ++++++++++--------
.../scudo/standalone/tests/primary_test.cpp | 34 ++--
4 files changed, 158 insertions(+), 139 deletions(-)
diff --git a/compiler-rt/lib/scudo/standalone/allocator_common.h b/compiler-rt/lib/scudo/standalone/allocator_common.h
index 95f4776ac596dc..46dc7c0f3b914e 100644
--- a/compiler-rt/lib/scudo/standalone/allocator_common.h
+++ b/compiler-rt/lib/scudo/standalone/allocator_common.h
@@ -40,6 +40,7 @@ template <class SizeClassAllocator> struct TransferBatch {
B->Count = static_cast<u16>(B->Count - N);
}
void clear() { Count = 0; }
+ bool empty() { return Count == 0; }
void add(CompactPtrT P) {
DCHECK_LT(Count, MaxNumCached);
Batch[Count++] = P;
@@ -48,6 +49,12 @@ template <class SizeClassAllocator> struct TransferBatch {
memcpy(Array, Batch, sizeof(Batch[0]) * Count);
clear();
}
+
+ void moveNToArray(CompactPtrT *Array, u16 N) {
+ DCHECK_LE(N, Count);
+ memcpy(Array, Batch + Count - N, sizeof(Batch[0]) * Count);
+ Count -= N;
+ }
u16 getCount() const { return Count; }
bool isEmpty() const { return Count == 0U; }
CompactPtrT get(u16 I) const {
diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index 8281e02ba164c1..d7cf09a376d504 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -191,38 +191,21 @@ template <typename Config> class SizeClassAllocator32 {
return BlockSize > PageSize;
}
- // Note that the `MaxBlockCount` will be used when we support arbitrary blocks
- // count. Now it's the same as the number of blocks stored in the
- // `TransferBatch`.
u16 popBlocks(CacheT *C, uptr ClassId, CompactPtrT *ToArray,
- UNUSED const u16 MaxBlockCount) {
- TransferBatchT *B = popBatch(C, ClassId);
- if (!B)
- return 0;
-
- const u16 Count = B->getCount();
- DCHECK_GT(Count, 0U);
- B->moveToArray(ToArray);
-
- if (ClassId != SizeClassMap::BatchClassId)
- C->deallocate(SizeClassMap::BatchClassId, B);
-
- return Count;
- }
-
- TransferBatchT *popBatch(CacheT *C, uptr ClassId) {
+ const u16 MaxBlockCount) {
DCHECK_LT(ClassId, NumClasses);
SizeClassInfo *Sci = getSizeClassInfo(ClassId);
ScopedLock L(Sci->Mutex);
- TransferBatchT *B = popBatchImpl(C, ClassId, Sci);
- if (UNLIKELY(!B)) {
+
+ u16 PopCount = popBlocksImpl(C, ClassId, Sci, ToArray, MaxBlockCount);
+ if (UNLIKELY(PopCount == 0)) {
if (UNLIKELY(!populateFreeList(C, ClassId, Sci)))
- return nullptr;
- B = popBatchImpl(C, ClassId, Sci);
- // if `populateFreeList` succeeded, we are supposed to get free blocks.
- DCHECK_NE(B, nullptr);
+ return 0U;
+ PopCount = popBlocksImpl(C, ClassId, Sci, ToArray, MaxBlockCount);
+ DCHECK_NE(PopCount, 0U);
}
- return B;
+
+ return PopCount;
}
// Push the array of free blocks to the designated batch group.
@@ -510,7 +493,7 @@ template <typename Config> class SizeClassAllocator32 {
// by TransferBatch is also free for use. We don't need to recycle the
// TransferBatch. Note that the correctness is maintained by the invariant,
//
- // The unit of each popBatch() request is entire TransferBatch. Return
+ // Each popBlocks() request returns the entire TransferBatch. Return
// part of the blocks in a TransferBatch is invalid.
//
// This ensures that TransferBatch won't leak the address itself while it's
@@ -634,7 +617,7 @@ template <typename Config> class SizeClassAllocator32 {
BG->Batches.push_front(TB);
BG->PushedBlocks = 0;
BG->BytesInBGAtLastCheckpoint = 0;
- BG->MaxCachedPerBatch = CacheT::getMaxCached(getSizeByClassId(ClassId));
+ BG->MaxCachedPerBatch = TransferBatchT::MaxNumCached;
return BG;
};
@@ -726,14 +709,11 @@ template <typename Config> class SizeClassAllocator32 {
InsertBlocks(Cur, Array + Size - Count, Count);
}
- // Pop one TransferBatch from a BatchGroup. The BatchGroup with the smallest
- // group id will be considered first.
- //
- // The region mutex needs to be held while calling this method.
- TransferBatchT *popBatchImpl(CacheT *C, uptr ClassId, SizeClassInfo *Sci)
+ u16 popBlocksImpl(CacheT *C, uptr ClassId, SizeClassInfo *Sci,
+ CompactPtrT *ToArray, const u16 MaxBlockCount)
REQUIRES(Sci->Mutex) {
if (Sci->FreeListInfo.BlockList.empty())
- return nullptr;
+ return 0U;
SinglyLinkedList<TransferBatchT> &Batches =
Sci->FreeListInfo.BlockList.front()->Batches;
@@ -746,33 +726,57 @@ template <typename Config> class SizeClassAllocator32 {
// Block used by `BatchGroup` is from BatchClassId. Turn the block into
// `TransferBatch` with single block.
TransferBatchT *TB = reinterpret_cast<TransferBatchT *>(BG);
- TB->clear();
- TB->add(
- compactPtr(SizeClassMap::BatchClassId, reinterpret_cast<uptr>(TB)));
+ ToArray[0] =
+ compactPtr(SizeClassMap::BatchClassId, reinterpret_cast<uptr>(TB));
Sci->FreeListInfo.PoppedBlocks += 1;
- return TB;
+ return 1U;
}
+ // So far, instead of always fill blocks to `MaxBlockCount`, we only exmaine
+ // single `TransferBatch` to minimize the time spent in the primary
+ // allocator. Besides, the sizes of `TransferBatch` and
+ // `CacheT::getMaxCached()` may also impact the times of accessing the
+ // primary allocator.
+ // TODO(chiahungduan): Evaluate if we want to always prepare `MaxBlockCount`
+ // blocks and/or adjust the size of `TransferBatch` according to
+ // `CacheT::getMaxCached()`.
TransferBatchT *B = Batches.front();
- Batches.pop_front();
DCHECK_NE(B, nullptr);
DCHECK_GT(B->getCount(), 0U);
- if (Batches.empty()) {
- BatchGroupT *BG = Sci->FreeListInfo.BlockList.front();
- Sci->FreeListInfo.BlockList.pop_front();
-
- // We don't keep BatchGroup with zero blocks to avoid empty-checking while
- // allocating. Note that block used by constructing BatchGroup is recorded
- // as free blocks in the last element of BatchGroup::Batches. Which means,
- // once we pop the last TransferBatch, the block is implicitly
- // deallocated.
+ // BachClassId should always take all blocks in the TransferBatch. Read the
+ // comment in `pushBatchClassBlocks()` for more details.
+ const u16 PopCount = ClassId == SizeClassMap::BatchClassId
+ ? B->getCount()
+ : Min(MaxBlockCount, B->getCount());
+ B->moveNToArray(ToArray, PopCount);
+
+ // TODO(chiahungduan): The deallocation of unused BatchClassId blocks can be
+ // done without holding `Mutex`.
+ if (B->empty()) {
+ Batches.pop_front();
+ // `TransferBatch` of BatchClassId is self-contained, no need to
+ // deallocate. Read the comment in `pushBatchClassBlocks()` for more
+ // details.
if (ClassId != SizeClassMap::BatchClassId)
- C->deallocate(SizeClassMap::BatchClassId, BG);
+ C->deallocate(SizeClassMap::BatchClassId, B);
+
+ if (Batches.empty()) {
+ BatchGroupT *BG = Sci->FreeListInfo.BlockList.front();
+ Sci->FreeListInfo.BlockList.pop_front();
+
+ // We don't keep BatchGroup with zero blocks to avoid empty-checking
+ // while allocating. Note that block used by constructing BatchGroup is
+ // recorded as free blocks in the last element of BatchGroup::Batches.
+ // Which means, once we pop the last TransferBatch, the block is
+ // implicitly deallocated.
+ if (ClassId != SizeClassMap::BatchClassId)
+ C->deallocate(SizeClassMap::BatchClassId, BG);
+ }
}
- Sci->FreeListInfo.PoppedBlocks += B->getCount();
- return B;
+ Sci->FreeListInfo.PoppedBlocks += PopCount;
+ return PopCount;
}
NOINLINE bool populateFreeList(CacheT *C, uptr ClassId, SizeClassInfo *Sci)
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index d1929ff7212f47..5fd49ee60e2824 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -221,41 +221,24 @@ template <typename Config> class SizeClassAllocator64 {
DCHECK_EQ(BlocksInUse, BatchClassUsedInFreeLists);
}
- // Note that the `MaxBlockCount` will be used when we support arbitrary blocks
- // count. Now it's the same as the number of blocks stored in the
- // `TransferBatch`.
u16 popBlocks(CacheT *C, uptr ClassId, CompactPtrT *ToArray,
- UNUSED const u16 MaxBlockCount) {
- TransferBatchT *B = popBatch(C, ClassId);
- if (!B)
- return 0;
-
- const u16 Count = B->getCount();
- DCHECK_GT(Count, 0U);
- B->moveToArray(ToArray);
-
- if (ClassId != SizeClassMap::BatchClassId)
- C->deallocate(SizeClassMap::BatchClassId, B);
-
- return Count;
- }
-
- TransferBatchT *popBatch(CacheT *C, uptr ClassId) {
+ const u16 MaxBlockCount) {
DCHECK_LT(ClassId, NumClasses);
RegionInfo *Region = getRegionInfo(ClassId);
+ u16 PopCount = 0;
{
ScopedLock L(Region->FLLock);
- TransferBatchT *B = popBatchImpl(C, ClassId, Region);
- if (LIKELY(B))
- return B;
+ PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
+ if (PopCount != 0U)
+ return PopCount;
}
bool ReportRegionExhausted = false;
- TransferBatchT *B = nullptr;
if (conditionVariableEnabled()) {
- B = popBatchWithCV(C, ClassId, Region, ReportRegionExhausted);
+ PopCount = popBlocksWithCV(C, ClassId, Region, ToArray, MaxBlockCount,
+ ReportRegionExhausted);
} else {
while (true) {
// When two threads compete for `Region->MMLock`, we only want one of
@@ -264,13 +247,15 @@ template <typename Config> class SizeClassAllocator64 {
ScopedLock ML(Region->MMLock);
{
ScopedLock FL(Region->FLLock);
- if ((B = popBatchImpl(C, ClassId, Region)))
- break;
+ PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
+ if (PopCount != 0U)
+ return PopCount;
}
const bool RegionIsExhausted = Region->Exhausted;
if (!RegionIsExhausted)
- B = populateFreeListAndPopBatch(C, ClassId, Region);
+ PopCount = populateFreeListAndPopBlocks(C, ClassId, Region, ToArray,
+ MaxBlockCount);
ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
break;
}
@@ -286,7 +271,7 @@ template <typename Config> class SizeClassAllocator64 {
reportOutOfBatchClass();
}
- return B;
+ return PopCount;
}
// Push the array of free blocks to the designated batch group.
@@ -640,7 +625,7 @@ template <typename Config> class SizeClassAllocator64 {
// by TransferBatch is also free for use. We don't need to recycle the
// TransferBatch. Note that the correctness is maintained by the invariant,
//
- // The unit of each popBatch() request is entire TransferBatch. Return
+ // Each popBlocks() request returns the entire TransferBatch. Return
// part of the blocks in a TransferBatch is invalid.
//
// This ensures that TransferBatch won't leak the address itself while it's
@@ -763,7 +748,7 @@ template <typename Config> class SizeClassAllocator64 {
BG->Batches.push_front(TB);
BG->PushedBlocks = 0;
BG->BytesInBGAtLastCheckpoint = 0;
- BG->MaxCachedPerBatch = CacheT::getMaxCached(getSizeByClassId(ClassId));
+ BG->MaxCachedPerBatch = TransferBatchT::MaxNumCached;
return BG;
};
@@ -855,9 +840,10 @@ template <typename Config> class SizeClassAllocator64 {
InsertBlocks(Cur, Array + Size - Count, Count);
}
- TransferBatchT *popBatchWithCV(CacheT *C, uptr ClassId, RegionInfo *Region,
- bool &ReportRegionExhausted) {
- TransferBatchT *B = nullptr;
+ u16 popBlocksWithCV(CacheT *C, uptr ClassId, RegionInfo *Region,
+ CompactPtrT *ToArray, const u16 MaxBlockCount,
+ bool &ReportRegionExhausted) {
+ u16 PopCount = 0;
while (true) {
// We only expect one thread doing the freelist refillment and other
@@ -878,7 +864,8 @@ template <typename Config> class SizeClassAllocator64 {
const bool RegionIsExhausted = Region->Exhausted;
if (!RegionIsExhausted)
- B = populateFreeListAndPopBatch(C, ClassId, Region);
+ PopCount = populateFreeListAndPopBlocks(C, ClassId, Region, ToArray,
+ MaxBlockCount);
ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
{
@@ -905,7 +892,8 @@ template <typename Config> class SizeClassAllocator64 {
// blocks were used up right after the refillment. Therefore, we have to
// check if someone is still populating the freelist.
ScopedLock FL(Region->FLLock);
- if (LIKELY(B = popBatchImpl(C, ClassId, Region)))
+ PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
+ if (PopCount != 0U)
break;
if (!Region->isPopulatingFreeList)
@@ -918,21 +906,19 @@ template <typename Config> class SizeClassAllocator64 {
// `pushBatchClassBlocks()` and `mergeGroupsToReleaseBack()`.
Region->FLLockCV.wait(Region->FLLock);
- if (LIKELY(B = popBatchImpl(C, ClassId, Region)))
+ PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
+ if (PopCount != 0U)
break;
}
- return B;
+ return PopCount;
}
- // Pop one TransferBatch from a BatchGroup. The BatchGroup with the smallest
- // group id will be considered first.
- //
- // The region mutex needs to be held while calling this method.
- TransferBatchT *popBatchImpl(CacheT *C, uptr ClassId, RegionInfo *Region)
+ u16 popBlocksImpl(CacheT *C, uptr ClassId, RegionInfo *Region,
+ CompactPtrT *ToArray, const u16 MaxBlockCount)
REQUIRES(Region->FLLock) {
if (Region->FreeListInfo.BlockList.empty())
- return nullptr;
+ return 0U;
SinglyLinkedList<TransferBatchT> &Batches =
Region->FreeListInfo.BlockList.front()->Batches;
@@ -945,39 +931,64 @@ template <typename Config> class SizeClassAllocator64 {
// Block used by `BatchGroup` is from BatchClassId. Turn the block into
// `TransferBatch` with single block.
TransferBatchT *TB = reinterpret_cast<TransferBatchT *>(BG);
- TB->clear();
- TB->add(
- compactPtr(SizeClassMap::BatchClassId, reinterpret_cast<uptr>(TB)));
+ ToArray[0] =
+ compactPtr(SizeClassMap::BatchClassId, reinterpret_cast<uptr>(TB));
Region->FreeListInfo.PoppedBlocks += 1;
- return TB;
+ return 1U;
}
+ // So far, instead of always fill blocks to `MaxBlockCount`, we only exmaine
+ // single `TransferBatch` to minimize the time spent in the primary
+ // allocator. Besides, the sizes of `TransferBatch` and
+ // `CacheT::getMaxCached()` may also impact the times of accessing the
+ // primary allocator.
+ // TODO(chiahungduan): Evaluate if we want to always prepare `MaxBlockCount`
+ // blocks and/or adjust the size of `TransferBatch` according to
+ // `CacheT::getMaxCached()`.
TransferBatchT *B = Batches.front();
- Batches.pop_front();
DCHECK_NE(B, nullptr);
DCHECK_GT(B->getCount(), 0U);
- if (Batches.empty()) {
- BatchGroupT *BG = Region->FreeListInfo.BlockList.front();
- Region->FreeListInfo.BlockList.pop_front();
-
- // We don't keep BatchGroup with zero blocks to avoid empty-checking while
- // allocating. Note that block used by constructing BatchGroup is recorded
- // as free blocks in the last element of BatchGroup::Batches. Which means,
- // once we pop the last TransferBatch, the block is implicitly
- // deallocated.
+ // BachClassId should always take all blocks in the TransferBatch. Read the
+ // comment in `pushBatchClassBlocks()` for more details.
+ const u16 PopCount = ClassId == SizeClassMap::BatchClassId
+ ? B->getCount()
+ : Min(MaxBlockCount, B->getCount());
+ B->moveNToArray(ToArray, PopCount);
+
+ // TODO(chiahungduan): The deallocation of unused BatchClassId blocks can be
+ // done without holding `FLLock`.
+ if (B->empty()) {
+ Batches.pop_front();
+ // `TransferBatch` of BatchClassId is self-contained, no need to
+ // deallocate. Read the comment in `pushBatchClassBlocks()` for more
+ // details.
if (ClassId != SizeClassMap::BatchClassId)
- C->deallocate(SizeClassMap::BatchClassId, BG);
+ C->deallocate(SizeClassMap::BatchClassId, B);
+
+ if (Batches.empty()) {
+ BatchGroupT *BG = Region->FreeListInfo.BlockList.front();
+ Region->FreeListInfo.BlockList.pop_front();
+
+ // We don't keep BatchGroup with zero blocks to avoid empty-checking
+ // while allocating. Note that block used by constructing BatchGroup is
+ // recorded as free blocks in the last element of BatchGroup::Batches.
+ // Which means, once we pop the last TransferBatch, the block is
+ // implicitly deallocated.
+ if (ClassId != SizeClassMap::BatchClassId)
+ C->deallocate(SizeClassMap::BatchClassId, BG);
+ }
}
- Region->FreeListInfo.PoppedBlocks += B->getCount();
+ Region->FreeListInfo.PoppedBlocks += PopCount;
- return B;
+ return PopCount;
}
- // Refill the freelist and return one batch.
- NOINLINE TransferBatchT *populateFreeListAndPopBatch(CacheT *C, uptr ClassId,
- RegionInfo *Region)
+ NOINLINE u16 populateFreeListAndPopBlocks(CacheT *C, uptr ClassId,
+ RegionInfo *Region,
+ CompactPtrT *ToArray,
+ const u16 MaxBlockCount)
REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
const uptr Size = getSizeByClassId(ClassId);
const u16 MaxCount = CacheT::getMaxCached(Size);
@@ -994,7 +1005,7 @@ template <typename Config> class SizeClassAllocator64 {
const uptr RegionBase = RegionBeg - getRegionBaseByClassId(ClassId);
if (UNLIKELY(RegionBase + MappedUser + MapSize > RegionSize)) {
Region->Exhausted = true;
- return nullptr;
+ return 0U;
}
if (UNLIKELY(!Region->MemMapInfo.MemMap.remap(
@@ -1002,7 +1013,7 @@ template <typename Config> class SizeClassAllocator64 {
MAP_ALLOWNOMEM | MAP_RESIZABLE |
(useMemoryTagging<Config>(Options.load()) ? MAP_MEMTAG
: 0)))) {
- return nullptr;
+ return 0U;
}
Region->MemMapInfo.MappedUser += MapSize;
C->getStats().add(StatMapped, MapSize);
@@ -1049,8 +1060,9 @@ template <typename Config> class SizeClassAllocator64 {
pushBatchClassBlocks(Region, ShuffleArray, NumberOfBlocks);
}
- TransferBatchT *B = popBatchImpl(C, ClassId, Region);
- DCHECK_NE(B, nullptr);
+ const u16 PopCount =
+ popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
+ DCHECK_NE(PopCount, 0U);
// Note that `PushedBlocks` and `PoppedBlocks` are supposed to only record
// the requests from `PushBlocks` and `PopBatch` which are external
@@ -1062,7 +1074,7 @@ template <typename Config> class SizeClassAllocator64 {
C->getStats().add(StatFree, AllocatedUser);
Region->MemMapInfo.AllocatedUser += AllocatedUser;
- return B;
+ return PopCount;
}
void getStats(ScopedString *Str, uptr ClassId, RegionInfo *Region)
@@ -1182,7 +1194,7 @@ template <typename Config> class SizeClassAllocator64 {
}
// Note that we have extracted the `GroupsToRelease` from region freelist.
- // It's safe to let pushBlocks()/popBatches() access the remaining region
+ // It's safe to let pushBlocks()/popBlocks() access the remaining region
// freelist. In the steps 3 and 4, we will temporarily release the FLLock
// and lock it again before step 5.
diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
index 18171511758a14..f64a5143b30d46 100644
--- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
@@ -237,7 +237,6 @@ struct SmallRegionsConfig {
// For the 32-bit one, it requires actually exhausting memory, so we skip it.
TEST(ScudoPrimaryTest, Primary64OOM) {
using Primary = scudo::SizeClassAllocator64<SmallRegionsConfig>;
- using TransferBatch = Primary::TransferBatchT;
Primary Allocator;
Allocator.init(/*ReleaseToOsInterval=*/-1);
typename Primary::CacheT Cache;
@@ -245,29 +244,26 @@ TEST(ScudoPrimaryTest, Primary64OOM) {
Stats.init();
Cache.init(&Stats, &Allocator);
bool AllocationFailed = false;
- std::vector<TransferBatch *> Batches;
+ std::vector<void *> Blocks;
const scudo::uptr ClassId = Primary::SizeClassMap::LargestClassId;
const scudo::uptr Size = Primary::getSizeByClassId(ClassId);
- typename Primary::CacheT::CompactPtrT Blocks[TransferBatch::MaxNumCached];
+ const scudo::u16 MaxCachedBlockCount = Primary::CacheT::getMaxCached(Size);
for (scudo::uptr I = 0; I < 10000U; I++) {
- TransferBatch *B = Allocator.popBatch(&Cache, ClassId);
- if (!B) {
- AllocationFailed = true;
- break;
+ for (scudo::uptr J = 0; J < MaxCachedBlockCount; ++J) {
+ void *Ptr = Cache.allocate(ClassId);
+ if (Ptr == nullptr) {
+ AllocationFailed = true;
+ break;
+ }
+ memset(Ptr, 'B', Size);
+ Blocks.push_back(Ptr);
}
- for (scudo::u16 J = 0; J < B->getCount(); J++)
- memset(Allocator.decompactPtr(ClassId, B->get(J)), 'B', Size);
- Batches.push_back(B);
- }
- while (!Batches.empty()) {
- TransferBatch *B = Batches.back();
- Batches.pop_back();
- const scudo::u16 Count = B->getCount();
- B->moveToArray(Blocks);
- Allocator.pushBlocks(&Cache, ClassId, Blocks, Count);
- Cache.deallocate(Primary::SizeClassMap::BatchClassId, B);
}
+
+ for (auto *Ptr : Blocks)
+ Cache.deallocate(ClassId, Ptr);
+
Cache.destroy(nullptr);
Allocator.releaseToOS(scudo::ReleaseToOS::Force);
scudo::ScopedString Str;
@@ -342,7 +338,7 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryThreaded) {
V.push_back(std::make_pair(ClassId, P));
}
- // Try to interleave pushBlocks(), popBatch() and releaseToOS().
+ // Try to interleave pushBlocks(), popBlocks() and releaseToOS().
Allocator->releaseToOS(scudo::ReleaseToOS::Force);
while (!V.empty()) {
>From 22678ca4fbb3d49b03c7873ed1e90bd6773fcfe2 Mon Sep 17 00:00:00 2001
From: Chia-hung Duan <chiahungduan at google.com>
Date: Tue, 31 Oct 2023 20:35:35 +0000
Subject: [PATCH 2/5] fixup! [scudo] Store more blocks in each TransferBatch
---
compiler-rt/lib/scudo/standalone/primary32.h | 2 +-
compiler-rt/lib/scudo/standalone/primary64.h | 3 +--
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index d7cf09a376d504..c44e0196143679 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -732,7 +732,7 @@ template <typename Config> class SizeClassAllocator32 {
return 1U;
}
- // So far, instead of always fill blocks to `MaxBlockCount`, we only exmaine
+ // So far, instead of always fill blocks to `MaxBlockCount`, we only examine
// single `TransferBatch` to minimize the time spent in the primary
// allocator. Besides, the sizes of `TransferBatch` and
// `CacheT::getMaxCached()` may also impact the times of accessing the
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 5fd49ee60e2824..80d56820057843 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -12,6 +12,7 @@
#include "allocator_common.h"
#include "bytemap.h"
#include "common.h"
+#include "condition_variable.h"
#include "list.h"
#include "local_cache.h"
#include "mem_map.h"
@@ -22,8 +23,6 @@
#include "string_utils.h"
#include "thread_annotations.h"
-#include "condition_variable.h"
-
namespace scudo {
// SizeClassAllocator64 is an allocator tuned for 64-bit address space.
>From 1e717dd6dbd75c652a3c9f1f219a390015dc8f7f Mon Sep 17 00:00:00 2001
From: Chia-hung Duan <chiahungduan at google.com>
Date: Thu, 15 Feb 2024 22:34:08 +0000
Subject: [PATCH 3/5] Address review comments
---
compiler-rt/lib/scudo/standalone/primary32.h | 10 +++++-----
compiler-rt/lib/scudo/standalone/primary64.h | 6 +++---
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index c44e0196143679..d08da179415e99 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -493,7 +493,7 @@ template <typename Config> class SizeClassAllocator32 {
// by TransferBatch is also free for use. We don't need to recycle the
// TransferBatch. Note that the correctness is maintained by the invariant,
//
- // Each popBlocks() request returns the entire TransferBatch. Return
+ // Each popBlocks() request returns the entire TransferBatch. Returning
// part of the blocks in a TransferBatch is invalid.
//
// This ensures that TransferBatch won't leak the address itself while it's
@@ -732,10 +732,10 @@ template <typename Config> class SizeClassAllocator32 {
return 1U;
}
- // So far, instead of always fill blocks to `MaxBlockCount`, we only examine
- // single `TransferBatch` to minimize the time spent in the primary
+ // So far, instead of always filling the blocks to `MaxBlockCount`, we only
+ // examine single `TransferBatch` to minimize the time spent on the primary
// allocator. Besides, the sizes of `TransferBatch` and
- // `CacheT::getMaxCached()` may also impact the times of accessing the
+ // `CacheT::getMaxCached()` may also impact the time spent on accessing the
// primary allocator.
// TODO(chiahungduan): Evaluate if we want to always prepare `MaxBlockCount`
// blocks and/or adjust the size of `TransferBatch` according to
@@ -766,7 +766,7 @@ template <typename Config> class SizeClassAllocator32 {
Sci->FreeListInfo.BlockList.pop_front();
// We don't keep BatchGroup with zero blocks to avoid empty-checking
- // while allocating. Note that block used by constructing BatchGroup is
+ // while allocating. Note that block used for constructing BatchGroup is
// recorded as free blocks in the last element of BatchGroup::Batches.
// Which means, once we pop the last TransferBatch, the block is
// implicitly deallocated.
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 80d56820057843..4c539e96dd2147 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -624,7 +624,7 @@ template <typename Config> class SizeClassAllocator64 {
// by TransferBatch is also free for use. We don't need to recycle the
// TransferBatch. Note that the correctness is maintained by the invariant,
//
- // Each popBlocks() request returns the entire TransferBatch. Return
+ // Each popBlocks() request returns the entire TransferBatch. Returning
// part of the blocks in a TransferBatch is invalid.
//
// This ensures that TransferBatch won't leak the address itself while it's
@@ -939,7 +939,7 @@ template <typename Config> class SizeClassAllocator64 {
// So far, instead of always fill blocks to `MaxBlockCount`, we only exmaine
// single `TransferBatch` to minimize the time spent in the primary
// allocator. Besides, the sizes of `TransferBatch` and
- // `CacheT::getMaxCached()` may also impact the times of accessing the
+ // `CacheT::getMaxCached()` may also impact the time spent on accessing the
// primary allocator.
// TODO(chiahungduan): Evaluate if we want to always prepare `MaxBlockCount`
// blocks and/or adjust the size of `TransferBatch` according to
@@ -970,7 +970,7 @@ template <typename Config> class SizeClassAllocator64 {
Region->FreeListInfo.BlockList.pop_front();
// We don't keep BatchGroup with zero blocks to avoid empty-checking
- // while allocating. Note that block used by constructing BatchGroup is
+ // while allocating. Note that block used for constructing BatchGroup is
// recorded as free blocks in the last element of BatchGroup::Batches.
// Which means, once we pop the last TransferBatch, the block is
// implicitly deallocated.
>From 0d967b495ef8bb4077c5023df56950eeadb5ddf0 Mon Sep 17 00:00:00 2001
From: Chia-hung Duan <chiahungduan at google.com>
Date: Sat, 10 Feb 2024 00:16:45 +0000
Subject: [PATCH 4/5] [scudo] Refactor allocator config to support optional
flags
Instead of explicitly disabling a feature by declaring the variable and
set it to false, this change supports the optional flags. I.e., you can
skip certain flags if you are not using it.
This optional feature supports both forms,
1. Value: A parameter for a feature. E.g., EnableRandomOffset
2. Type: A C++ type implementing a feature. E.g., ConditionVariableT
On the other hand, to access the flags will be through one of the
wrappers, BaseConfig/PrimaryConfig/SecondaryConfig/CacheConfig
(CacheConfig is embedded in SecondaryConfig). These wrappers have the
getters to access the value and the type. When adding a new feature, we
need to add it to `allocator_config.def` and mark the new variable with
either *_REQUIRED_* or *_OPTIONAL_* macro so that the accessor will be
generated properly.
In addition, also remove the need of `UseConditionVariable` to flip
on/off of condition variable. Now we only need to define the type of
condition variable.
---
.../lib/scudo/standalone/CMakeLists.txt | 1 +
.../lib/scudo/standalone/allocator_config.def | 123 ++++++++++++++++
.../lib/scudo/standalone/allocator_config.h | 78 +---------
.../standalone/allocator_config_wrapper.h | 134 ++++++++++++++++++
compiler-rt/lib/scudo/standalone/combined.h | 54 +++----
.../lib/scudo/standalone/condition_variable.h | 16 ---
compiler-rt/lib/scudo/standalone/memtag.h | 2 +-
compiler-rt/lib/scudo/standalone/primary32.h | 25 ++--
compiler-rt/lib/scudo/standalone/primary64.h | 25 ++--
compiler-rt/lib/scudo/standalone/secondary.h | 36 +++--
.../scudo/standalone/tests/combined_test.cpp | 1 -
.../scudo/standalone/tests/primary_test.cpp | 29 +++-
.../scudo/standalone/tests/secondary_test.cpp | 22 ++-
13 files changed, 378 insertions(+), 168 deletions(-)
create mode 100644 compiler-rt/lib/scudo/standalone/allocator_config.def
create mode 100644 compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h
diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
index 60092005cc33bb..6fb4e88de3155f 100644
--- a/compiler-rt/lib/scudo/standalone/CMakeLists.txt
+++ b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
@@ -58,6 +58,7 @@ endif()
set(SCUDO_HEADERS
allocator_common.h
allocator_config.h
+ allocator_config_wrapper.h
atomic_helpers.h
bytemap.h
checksum.h
diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.def b/compiler-rt/lib/scudo/standalone/allocator_config.def
new file mode 100644
index 00000000000000..ddde71d2463f78
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/allocator_config.def
@@ -0,0 +1,123 @@
+//===-- allocator_config.def ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines all the flags and types supported in Scudo.
+
+#ifndef BASE_REQUIRED_TEMPLATE_TYPE
+#define BASE_REQUIRED_TEMPLATE_TYPE(...)
+#endif
+#ifndef BASE_OPTIONAL
+#define BASE_OPTIONAL(...)
+#endif
+#ifndef PRIMARY_REQUIRED_TYPE
+#define PRIMARY_REQUIRED_TYPE(...)
+#endif
+#ifndef PRIMARY_REQUIRED
+#define PRIMARY_REQUIRED(...)
+#endif
+#ifndef PRIMARY_OPTIONAL
+#define PRIMARY_OPTIONAL(...)
+#endif
+#ifndef PRIMARY_OPTIONAL_TYPE
+#define PRIMARY_OPTIONAL_TYPE(...)
+#endif
+#ifndef SECONDARY_REQUIRED_TEMPLATE_TYPE
+#define SECONDARY_REQUIRED_TEMPLATE_TYPE(...)
+#endif
+#ifndef SECONDARY_CACHE_OPTIONAL
+#define SECONDARY_CACHE_OPTIONAL(...)
+#endif
+
+// BASE_REQUIRED_TEMPLATE_TYPE(NAME)
+//
+// Thread-Specific Data Registry used, shared or exclusive.
+BASE_REQUIRED_TEMPLATE_TYPE(TSDRegistryT)
+
+// Defines the type of Primary allocator to use.
+BASE_REQUIRED_TEMPLATE_TYPE(PrimaryT)
+
+// Defines the type of Secondary allocator to use.
+BASE_REQUIRED_TEMPLATE_TYPE(SecondaryT)
+
+// BASE_OPTIONAL(TYPE, NAME, DEFAULT)
+//
+// Indicates possible support for Memory Tagging.
+BASE_OPTIONAL(const bool, MaySupportMemoryTagging, false)
+
+// PRIMARY_REQUIRED_TYPE(NAME)
+//
+// SizeClassMap to use with the Primary.
+PRIMARY_REQUIRED_TYPE(SizeClassMap)
+
+// Defines the type and scale of a compact pointer. A compact pointer can
+// be understood as the offset of a pointer within the region it belongs
+// to, in increments of a power-of-2 scale. See `CompactPtrScale` also.
+PRIMARY_REQUIRED_TYPE(CompactPtrT)
+
+// PRIMARY_REQUIRED(TYPE, NAME)
+//
+// The scale of a compact pointer. E.g., Ptr = Base + (CompactPtr << Scale).
+PRIMARY_REQUIRED(const uptr, CompactPtrScale)
+
+// Log2 of the size of a size class region, as used by the Primary.
+PRIMARY_REQUIRED(const uptr, RegionSizeLog)
+
+// Log2 of the size of block group, as used by the Primary. Each group
+// contains a range of memory addresses, blocks in the range will belong
+// to the same group. In general, single region may have 1 or 2MB group
+// size. Multiple regions will have the group size equal to the region
+// size because the region size is usually smaller than 1 MB.
+// Smaller value gives fine-grained control of memory usage but the
+// trade-off is that it may take longer time of deallocation.
+PRIMARY_REQUIRED(const uptr, GroupSizeLog)
+
+// Call map for user memory with at least this size. Only used with primary64.
+PRIMARY_REQUIRED(const uptr, MapSizeIncrement)
+
+// Defines the minimal & maximal release interval that can be set.
+PRIMARY_REQUIRED(const s32, MinReleaseToOsIntervalMs)
+PRIMARY_REQUIRED(const s32, MaxReleaseToOsIntervalMs)
+
+// PRIMARY_OPTIONAL(TYPE, NAME, DEFAULT)
+//
+// Indicates support for offsetting the start of a region by a random number of
+// pages. Only used with primary64.
+PRIMARY_OPTIONAL(const bool, EnableRandomOffset, false)
+
+// PRIMARY_OPTIONAL_TYPE(NAME, DEFAULT)
+//
+// Use condition variable to shorten the waiting time of refillment of
+// freelist. Note that this depends on the implementation of condition
+// variable on each platform and the performance may vary so that it doesn not
+// guarantee a performance benefit.
+PRIMARY_OPTIONAL_TYPE(ConditionVariableT, ConditionVariableDummy)
+
+// SECONDARY_REQUIRED_TEMPLATE_TYPE(NAME)
+//
+// Defines the type of Secondary Cache to use.
+SECONDARY_REQUIRED_TEMPLATE_TYPE(CacheT)
+
+// SECONDARY_CACHE_OPTIONAL(TYPE, NAME, DEFAULT)
+//
+// Defines the type of cache used by the Secondary. Some additional
+// configuration entries can be necessary depending on the Cache.
+SECONDARY_CACHE_OPTIONAL(const u32, EntriesArraySize, 0)
+SECONDARY_CACHE_OPTIONAL(const u32, QuarantineSize, 0)
+SECONDARY_CACHE_OPTIONAL(const u32, DefaultMaxEntriesCount, 0)
+SECONDARY_CACHE_OPTIONAL(const u32, DefaultMaxEntrySize, 0)
+SECONDARY_CACHE_OPTIONAL(const s32, MinReleaseToOsIntervalMs, INT32_MIN)
+SECONDARY_CACHE_OPTIONAL(const s32, MaxReleaseToOsIntervalMs, INT32_MAX)
+
+#undef SECONDARY_CACHE_OPTIONAL
+#undef SECONDARY_REQUIRED_TEMPLATE_TYPE
+#undef PRIMARY_OPTIONAL_TYPE
+#undef PRIMARY_OPTIONAL
+#undef PRIMARY_REQUIRED
+#undef PRIMARY_REQUIRED_TYPE
+#undef BASE_OPTIONAL
+#undef BASE_REQUIRED_TEMPLATE_TYPE
diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.h b/compiler-rt/lib/scudo/standalone/allocator_config.h
index 3c6aa3acb0e45b..1e0cf1015ba67e 100644
--- a/compiler-rt/lib/scudo/standalone/allocator_config.h
+++ b/compiler-rt/lib/scudo/standalone/allocator_config.h
@@ -38,80 +38,10 @@
namespace scudo {
-// The combined allocator uses a structure as a template argument that
-// specifies the configuration options for the various subcomponents of the
-// allocator.
-//
-// struct ExampleConfig {
-// // Indicates possible support for Memory Tagging.
-// static const bool MaySupportMemoryTagging = false;
-//
-// // Thread-Specific Data Registry used, shared or exclusive.
-// template <class A> using TSDRegistryT = TSDRegistrySharedT<A, 8U, 4U>;
-//
-// struct Primary {
-// // SizeClassMap to use with the Primary.
-// using SizeClassMap = DefaultSizeClassMap;
-//
-// // Log2 of the size of a size class region, as used by the Primary.
-// static const uptr RegionSizeLog = 30U;
-//
-// // Log2 of the size of block group, as used by the Primary. Each group
-// // contains a range of memory addresses, blocks in the range will belong
-// // to the same group. In general, single region may have 1 or 2MB group
-// // size. Multiple regions will have the group size equal to the region
-// // size because the region size is usually smaller than 1 MB.
-// // Smaller value gives fine-grained control of memory usage but the
-// // trade-off is that it may take longer time of deallocation.
-// static const uptr GroupSizeLog = 20U;
-//
-// // Defines the type and scale of a compact pointer. A compact pointer can
-// // be understood as the offset of a pointer within the region it belongs
-// // to, in increments of a power-of-2 scale.
-// // eg: Ptr = Base + (CompactPtr << Scale).
-// typedef u32 CompactPtrT;
-// static const uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
-//
-// // Indicates support for offsetting the start of a region by
-// // a random number of pages. Only used with primary64.
-// static const bool EnableRandomOffset = true;
-//
-// // Call map for user memory with at least this size. Only used with
-// // primary64.
-// static const uptr MapSizeIncrement = 1UL << 18;
-//
-// // Defines the minimal & maximal release interval that can be set.
-// static const s32 MinReleaseToOsIntervalMs = INT32_MIN;
-// static const s32 MaxReleaseToOsIntervalMs = INT32_MAX;
-//
-// // Use condition variable to shorten the waiting time of refillment of
-// // freelist. Note that this depends on the implementation of condition
-// // variable on each platform and the performance may vary so that it
-// // doesn't guarantee a performance benefit.
-// // Note that both variables have to be defined to enable it.
-// static const bool UseConditionVariable = true;
-// using ConditionVariableT = ConditionVariableLinux;
-// };
-// // Defines the type of Primary allocator to use.
-// template <typename Config> using PrimaryT = SizeClassAllocator64<Config>;
-//
-// // Defines the type of cache used by the Secondary. Some additional
-// // configuration entries can be necessary depending on the Cache.
-// struct Secondary {
-// struct Cache {
-// static const u32 EntriesArraySize = 32U;
-// static const u32 QuarantineSize = 0U;
-// static const u32 DefaultMaxEntriesCount = 32U;
-// static const uptr DefaultMaxEntrySize = 1UL << 19;
-// static const s32 MinReleaseToOsIntervalMs = INT32_MIN;
-// static const s32 MaxReleaseToOsIntervalMs = INT32_MAX;
-// };
-// // Defines the type of Secondary Cache to use.
-// template <typename Config> using CacheT = MapAllocatorCache<Config>;
-// };
-// // Defines the type of Secondary allocator to use.
-// template <typename Config> using SecondaryT = MapAllocator<Config>;
-// };
+// Scudo uses a structure as a template argument that specifies the
+// configuration options for the various subcomponents of the allocator. See the
+// following configs as examples and check `allocator_config.def` for all the
+// available options.
#ifndef SCUDO_USE_CUSTOM_CONFIG
diff --git a/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h
new file mode 100644
index 00000000000000..c946531dc4e02f
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h
@@ -0,0 +1,134 @@
+//===-- allocator_config_wrapper.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_ALLOCATOR_CONFIG_WRAPPER_H_
+#define SCUDO_ALLOCATOR_CONFIG_WRAPPER_H_
+
+#include "condition_variable.h"
+#include "internal_defs.h"
+#include "secondary.h"
+
+namespace {
+
+template <typename T> struct removeConst {
+ using type = T;
+};
+template <typename T> struct removeConst<const T> {
+ using type = T;
+};
+
+} // namespace
+
+namespace scudo {
+
+#define OPTIONAL_TEMPLATE(TYPE, NAME, DEFAULT) \
+ template <typename Config, typename = TYPE> struct NAME##State { \
+ static constexpr removeConst<TYPE>::type getValue() { return DEFAULT; } \
+ }; \
+ template <typename Config> \
+ struct NAME##State<Config, decltype(Config::NAME)> { \
+ static constexpr removeConst<TYPE>::type getValue() { \
+ return Config::NAME; \
+ } \
+ };
+
+#define OPTIONAL_TYPE_TEMPLATE(NAME, DEFAULT) \
+ template <typename Config, typename Void = Config> struct NAME##Type { \
+ static constexpr bool enabled() { return false; } \
+ using NAME = DEFAULT; \
+ }; \
+ template <typename Config> \
+ struct NAME##Type<Config, typename Config::NAME> { \
+ static constexpr bool enabled() { return true; } \
+ using NAME = typename Config::NAME; \
+ };
+
+template <typename AllocatorConfig> struct BaseConfig {
+#define BASE_REQUIRED_TEMPLATE_TYPE(NAME) \
+ template <typename T> using NAME = typename AllocatorConfig::template NAME<T>;
+
+#define BASE_OPTIONAL(TYPE, NAME, DEFAULT) \
+ OPTIONAL_TEMPLATE(TYPE, NAME, DEFAULT) \
+ static constexpr removeConst<TYPE>::type get##NAME() { \
+ return NAME##State<AllocatorConfig>::getValue(); \
+ }
+
+#include "allocator_config.def"
+}; // BaseConfig
+
+template <typename AllocatorConfig> struct PrimaryConfig {
+ // TODO: Pass this flag through template argument to remove this hard-coded
+ // function.
+ static constexpr bool getMaySupportMemoryTagging() {
+ return BaseConfig<AllocatorConfig>::getMaySupportMemoryTagging();
+ }
+
+#define PRIMARY_REQUIRED_TYPE(NAME) \
+ using NAME = typename AllocatorConfig::Primary::NAME;
+
+#define PRIMARY_REQUIRED(TYPE, NAME) \
+ static constexpr removeConst<TYPE>::type get##NAME() { \
+ return AllocatorConfig::Primary::NAME; \
+ }
+
+#define PRIMARY_OPTIONAL(TYPE, NAME, DEFAULT) \
+ OPTIONAL_TEMPLATE(TYPE, NAME, DEFAULT) \
+ static constexpr removeConst<TYPE>::type get##NAME() { \
+ return NAME##State<typename AllocatorConfig::Primary>::getValue(); \
+ }
+
+#define PRIMARY_OPTIONAL_TYPE(NAME, DEFAULT) \
+ OPTIONAL_TYPE_TEMPLATE(NAME, DEFAULT) \
+ static constexpr bool has##NAME() { \
+ return NAME##Type<typename AllocatorConfig::Primary>::enabled(); \
+ } \
+ using NAME = typename NAME##Type<typename AllocatorConfig::Primary>::NAME;
+
+#include "allocator_config.def"
+
+}; // PrimaryConfig
+
+template <typename AllocatorConfig> struct SecondaryConfig {
+ // TODO: Pass this flag through template argument to remove this hard-coded
+ // function.
+ static constexpr bool getMaySupportMemoryTagging() {
+ return BaseConfig<AllocatorConfig>::getMaySupportMemoryTagging();
+ }
+
+#define SECONDARY_REQUIRED_TEMPLATE_TYPE(NAME) \
+ template <typename T> \
+ using NAME = typename AllocatorConfig::Secondary::template NAME<T>;
+#include "allocator_config.def"
+
+ struct CacheConfig {
+ // TODO: Pass this flag through template argument to remove this hard-coded
+ // function.
+ static constexpr bool getMaySupportMemoryTagging() {
+ return BaseConfig<AllocatorConfig>::getMaySupportMemoryTagging();
+ }
+
+#define SECONDARY_CACHE_OPTIONAL(TYPE, NAME, DEFAULT) \
+ template <typename Config, typename = TYPE> struct NAME##State { \
+ static constexpr removeConst<TYPE>::type getValue() { return DEFAULT; } \
+ }; \
+ template <typename Config> \
+ struct NAME##State<Config, decltype(Config::Cache)> { \
+ static constexpr removeConst<TYPE>::type getValue() { \
+ return Config::Cache::NAME; \
+ } \
+ }; \
+ static constexpr removeConst<TYPE>::type get##NAME() { \
+ return NAME##State<typename AllocatorConfig::Secondary>::getValue(); \
+ }
+#include "allocator_config.def"
+ };
+}; // SecondaryConfig
+
+} // namespace scudo
+
+#endif // SCUDO_ALLOCATOR_CONFIG_WRAPPER_H_
diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index b1700e5ecef7f5..e53d19b0777a44 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -9,6 +9,7 @@
#ifndef SCUDO_COMBINED_H_
#define SCUDO_COMBINED_H_
+#include "allocator_config_wrapper.h"
#include "chunk.h"
#include "common.h"
#include "flags.h"
@@ -45,11 +46,14 @@ namespace scudo {
template <class Config, void (*PostInitCallback)(void) = EmptyCallback>
class Allocator {
public:
- using PrimaryT = typename Config::template PrimaryT<Config>;
- using SecondaryT = typename Config::template SecondaryT<Config>;
+ using AllocatorConfig = BaseConfig<Config>;
+ using PrimaryT =
+ typename AllocatorConfig::template PrimaryT<PrimaryConfig<Config>>;
+ using SecondaryT =
+ typename AllocatorConfig::template SecondaryT<SecondaryConfig<Config>>;
using CacheT = typename PrimaryT::CacheT;
typedef Allocator<Config, PostInitCallback> ThisT;
- typedef typename Config::template TSDRegistryT<ThisT> TSDRegistryT;
+ typedef typename AllocatorConfig::template TSDRegistryT<ThisT> TSDRegistryT;
void callPostInitCallback() {
pthread_once(&PostInitNonce, PostInitCallback);
@@ -70,7 +74,7 @@ class Allocator {
Header.State = Chunk::State::Available;
Chunk::storeHeader(Allocator.Cookie, Ptr, &Header);
- if (allocatorSupportsMemoryTagging<Config>())
+ if (allocatorSupportsMemoryTagging<AllocatorConfig>())
Ptr = untagPointer(Ptr);
void *BlockBegin = Allocator::getBlockBegin(Ptr, &Header);
Cache.deallocate(Header.ClassId, BlockBegin);
@@ -97,7 +101,8 @@ class Allocator {
// Reset tag to 0 as this chunk may have been previously used for a tagged
// user allocation.
- if (UNLIKELY(useMemoryTagging<Config>(Allocator.Primary.Options.load())))
+ if (UNLIKELY(useMemoryTagging<AllocatorConfig>(
+ Allocator.Primary.Options.load())))
storeTags(reinterpret_cast<uptr>(Ptr),
reinterpret_cast<uptr>(Ptr) + sizeof(QuarantineBatch));
@@ -157,7 +162,7 @@ class Allocator {
Primary.Options.set(OptionBit::DeallocTypeMismatch);
if (getFlags()->delete_size_mismatch)
Primary.Options.set(OptionBit::DeleteSizeMismatch);
- if (allocatorSupportsMemoryTagging<Config>() &&
+ if (allocatorSupportsMemoryTagging<AllocatorConfig>() &&
systemSupportsMemoryTagging())
Primary.Options.set(OptionBit::UseMemoryTagging);
@@ -260,7 +265,7 @@ class Allocator {
void drainCaches() { TSDRegistry.drainCaches(this); }
ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) {
- if (!allocatorSupportsMemoryTagging<Config>())
+ if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
return Ptr;
auto UntaggedPtr = untagPointer(Ptr);
if (UntaggedPtr != Ptr)
@@ -272,7 +277,7 @@ class Allocator {
}
ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) {
- if (!allocatorSupportsMemoryTagging<Config>())
+ if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
return Ptr;
return addFixedTag(Ptr, 2);
}
@@ -409,7 +414,7 @@ class Allocator {
//
// When memory tagging is enabled, zeroing the contents is done as part of
// setting the tag.
- if (UNLIKELY(useMemoryTagging<Config>(Options))) {
+ if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
uptr PrevUserPtr;
Chunk::UnpackedHeader Header;
const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
@@ -491,7 +496,7 @@ class Allocator {
} else {
Block = addHeaderTag(Block);
Ptr = addHeaderTag(Ptr);
- if (UNLIKELY(useMemoryTagging<Config>(Options))) {
+ if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
storeTags(reinterpret_cast<uptr>(Block), reinterpret_cast<uptr>(Ptr));
storeSecondaryAllocationStackMaybe(Options, Ptr, Size);
}
@@ -651,7 +656,7 @@ class Allocator {
(reinterpret_cast<uptr>(OldTaggedPtr) + NewSize)) &
Chunk::SizeOrUnusedBytesMask;
Chunk::storeHeader(Cookie, OldPtr, &Header);
- if (UNLIKELY(useMemoryTagging<Config>(Options))) {
+ if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
if (ClassId) {
resizeTaggedChunk(reinterpret_cast<uptr>(OldTaggedPtr) + OldSize,
reinterpret_cast<uptr>(OldTaggedPtr) + NewSize,
@@ -752,8 +757,9 @@ class Allocator {
Base = untagPointer(Base);
const uptr From = Base;
const uptr To = Base + Size;
- bool MayHaveTaggedPrimary = allocatorSupportsMemoryTagging<Config>() &&
- systemSupportsMemoryTagging();
+ bool MayHaveTaggedPrimary =
+ allocatorSupportsMemoryTagging<AllocatorConfig>() &&
+ systemSupportsMemoryTagging();
auto Lambda = [this, From, To, MayHaveTaggedPrimary, Callback,
Arg](uptr Block) {
if (Block < From || Block >= To)
@@ -774,9 +780,9 @@ class Allocator {
}
if (Header.State == Chunk::State::Allocated) {
uptr TaggedChunk = Chunk;
- if (allocatorSupportsMemoryTagging<Config>())
+ if (allocatorSupportsMemoryTagging<AllocatorConfig>())
TaggedChunk = untagPointer(TaggedChunk);
- if (useMemoryTagging<Config>(Primary.Options.load()))
+ if (useMemoryTagging<AllocatorConfig>(Primary.Options.load()))
TaggedChunk = loadTag(Chunk);
Callback(TaggedChunk, getSize(reinterpret_cast<void *>(Chunk), &Header),
Arg);
@@ -868,7 +874,7 @@ class Allocator {
}
bool useMemoryTaggingTestOnly() const {
- return useMemoryTagging<Config>(Primary.Options.load());
+ return useMemoryTagging<AllocatorConfig>(Primary.Options.load());
}
void disableMemoryTagging() {
// If we haven't been initialized yet, we need to initialize now in order to
@@ -878,7 +884,7 @@ class Allocator {
// callback), which may cause mappings to be created with memory tagging
// enabled.
TSDRegistry.initOnceMaybe(this);
- if (allocatorSupportsMemoryTagging<Config>()) {
+ if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {
Secondary.disableMemoryTagging();
Primary.Options.clear(OptionBit::UseMemoryTagging);
}
@@ -962,7 +968,7 @@ class Allocator {
const char *Memory, const char *MemoryTags,
uintptr_t MemoryAddr, size_t MemorySize) {
*ErrorInfo = {};
- if (!allocatorSupportsMemoryTagging<Config>() ||
+ if (!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
MemoryAddr + MemorySize < MemoryAddr)
return;
@@ -1001,7 +1007,7 @@ class Allocator {
static_assert(MinAlignment >= sizeof(Chunk::PackedHeader),
"Minimal alignment must at least cover a chunk header.");
- static_assert(!allocatorSupportsMemoryTagging<Config>() ||
+ static_assert(!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
MinAlignment >= archMemoryTagGranuleSize(),
"");
@@ -1101,7 +1107,7 @@ class Allocator {
const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
if (LIKELY(Header->ClassId))
return SizeOrUnusedBytes;
- if (allocatorSupportsMemoryTagging<Config>())
+ if (allocatorSupportsMemoryTagging<AllocatorConfig>())
Ptr = untagPointer(const_cast<void *>(Ptr));
return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -
reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
@@ -1121,12 +1127,12 @@ class Allocator {
Header->State = Chunk::State::Available;
else
Header->State = Chunk::State::Quarantined;
- Header->OriginOrWasZeroed = useMemoryTagging<Config>(Options) &&
+ Header->OriginOrWasZeroed = useMemoryTagging<AllocatorConfig>(Options) &&
Header->ClassId &&
!TSDRegistry.getDisableMemInit();
Chunk::storeHeader(Cookie, Ptr, Header);
- if (UNLIKELY(useMemoryTagging<Config>(Options))) {
+ if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
u8 PrevTag = extractTag(reinterpret_cast<uptr>(TaggedPtr));
storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size);
if (Header->ClassId) {
@@ -1143,7 +1149,7 @@ class Allocator {
}
}
if (BypassQuarantine) {
- if (allocatorSupportsMemoryTagging<Config>())
+ if (allocatorSupportsMemoryTagging<AllocatorConfig>())
Ptr = untagPointer(Ptr);
void *BlockBegin = getBlockBegin(Ptr, Header);
const uptr ClassId = Header->ClassId;
@@ -1162,7 +1168,7 @@ class Allocator {
if (CacheDrained)
Primary.tryReleaseToOS(ClassId, ReleaseToOS::Normal);
} else {
- if (UNLIKELY(useMemoryTagging<Config>(Options)))
+ if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options)))
storeTags(reinterpret_cast<uptr>(BlockBegin),
reinterpret_cast<uptr>(Ptr));
Secondary.deallocate(Options, BlockBegin);
diff --git a/compiler-rt/lib/scudo/standalone/condition_variable.h b/compiler-rt/lib/scudo/standalone/condition_variable.h
index 549f6e9f787bad..3f16c86651e736 100644
--- a/compiler-rt/lib/scudo/standalone/condition_variable.h
+++ b/compiler-rt/lib/scudo/standalone/condition_variable.h
@@ -39,22 +39,6 @@ class ConditionVariableDummy
}
};
-template <typename Config, typename = const bool>
-struct ConditionVariableState {
- static constexpr bool enabled() { return false; }
- // This is only used for compilation purpose so that we won't end up having
- // many conditional compilations. If you want to use `ConditionVariableDummy`,
- // define `ConditionVariableT` in your allocator configuration. See
- // allocator_config.h for more details.
- using ConditionVariableT = ConditionVariableDummy;
-};
-
-template <typename Config>
-struct ConditionVariableState<Config, decltype(Config::UseConditionVariable)> {
- static constexpr bool enabled() { return true; }
- using ConditionVariableT = typename Config::ConditionVariableT;
-};
-
} // namespace scudo
#endif // SCUDO_CONDITION_VARIABLE_H_
diff --git a/compiler-rt/lib/scudo/standalone/memtag.h b/compiler-rt/lib/scudo/standalone/memtag.h
index aaed2192ad7521..1f6983e99404a2 100644
--- a/compiler-rt/lib/scudo/standalone/memtag.h
+++ b/compiler-rt/lib/scudo/standalone/memtag.h
@@ -326,7 +326,7 @@ inline void *addFixedTag(void *Ptr, uptr Tag) {
template <typename Config>
inline constexpr bool allocatorSupportsMemoryTagging() {
- return archSupportsMemoryTagging() && Config::MaySupportMemoryTagging &&
+ return archSupportsMemoryTagging() && Config::getMaySupportMemoryTagging() &&
(1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize();
}
diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index d08da179415e99..b5165fd383bd33 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -43,14 +43,13 @@ namespace scudo {
template <typename Config> class SizeClassAllocator32 {
public:
- typedef typename Config::Primary::CompactPtrT CompactPtrT;
- typedef typename Config::Primary::SizeClassMap SizeClassMap;
- static const uptr GroupSizeLog = Config::Primary::GroupSizeLog;
+ typedef typename Config::CompactPtrT CompactPtrT;
+ typedef typename Config::SizeClassMap SizeClassMap;
+ static const uptr GroupSizeLog = Config::getGroupSizeLog();
// The bytemap can only track UINT8_MAX - 1 classes.
static_assert(SizeClassMap::LargestClassId <= (UINT8_MAX - 1), "");
// Regions should be large enough to hold the largest Block.
- static_assert((1UL << Config::Primary::RegionSizeLog) >=
- SizeClassMap::MaxSize,
+ static_assert((1UL << Config::getRegionSizeLog()) >= SizeClassMap::MaxSize,
"");
typedef SizeClassAllocator32<Config> ThisT;
typedef SizeClassAllocatorLocalCache<ThisT> CacheT;
@@ -331,9 +330,9 @@ template <typename Config> class SizeClassAllocator32 {
bool setOption(Option O, sptr Value) {
if (O == Option::ReleaseInterval) {
- const s32 Interval = Max(Min(static_cast<s32>(Value),
- Config::Primary::MaxReleaseToOsIntervalMs),
- Config::Primary::MinReleaseToOsIntervalMs);
+ const s32 Interval = Max(
+ Min(static_cast<s32>(Value), Config::getMaxReleaseToOsIntervalMs()),
+ Config::getMinReleaseToOsIntervalMs());
atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval);
return true;
}
@@ -373,9 +372,9 @@ template <typename Config> class SizeClassAllocator32 {
private:
static const uptr NumClasses = SizeClassMap::NumClasses;
- static const uptr RegionSize = 1UL << Config::Primary::RegionSizeLog;
- static const uptr NumRegions =
- SCUDO_MMAP_RANGE_SIZE >> Config::Primary::RegionSizeLog;
+ static const uptr RegionSize = 1UL << Config::getRegionSizeLog();
+ static const uptr NumRegions = SCUDO_MMAP_RANGE_SIZE >>
+ Config::getRegionSizeLog();
static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U;
typedef FlatByteMap<NumRegions> ByteMap;
@@ -408,7 +407,7 @@ template <typename Config> class SizeClassAllocator32 {
static_assert(sizeof(SizeClassInfo) % SCUDO_CACHE_LINE_SIZE == 0, "");
uptr computeRegionId(uptr Mem) {
- const uptr Id = Mem >> Config::Primary::RegionSizeLog;
+ const uptr Id = Mem >> Config::getRegionSizeLog();
CHECK_LT(Id, NumRegions);
return Id;
}
@@ -437,7 +436,7 @@ template <typename Config> class SizeClassAllocator32 {
unmap(reinterpret_cast<void *>(End), MapEnd - End);
DCHECK_EQ(Region % RegionSize, 0U);
- static_assert(Config::Primary::RegionSizeLog == GroupSizeLog,
+ static_assert(Config::getRegionSizeLog() == GroupSizeLog,
"Memory group should be the same size as Region");
return Region;
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 4c539e96dd2147..c53e8f98547a18 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -47,13 +47,16 @@ namespace scudo {
template <typename Config> class SizeClassAllocator64 {
public:
- typedef typename Config::Primary::CompactPtrT CompactPtrT;
- typedef typename Config::Primary::SizeClassMap SizeClassMap;
+ typedef typename Config::CompactPtrT CompactPtrT;
+ typedef typename Config::SizeClassMap SizeClassMap;
+#if 0
typedef typename ConditionVariableState<
typename Config::Primary>::ConditionVariableT ConditionVariableT;
- static const uptr CompactPtrScale = Config::Primary::CompactPtrScale;
- static const uptr RegionSizeLog = Config::Primary::RegionSizeLog;
- static const uptr GroupSizeLog = Config::Primary::GroupSizeLog;
+#endif
+ typedef typename Config::ConditionVariableT ConditionVariableT;
+ static const uptr CompactPtrScale = Config::getCompactPtrScale();
+ static const uptr RegionSizeLog = Config::getRegionSizeLog();
+ static const uptr GroupSizeLog = Config::getGroupSizeLog();
static_assert(RegionSizeLog >= GroupSizeLog,
"Group size shouldn't be greater than the region size");
static const uptr GroupScale = GroupSizeLog - CompactPtrScale;
@@ -74,7 +77,7 @@ template <typename Config> class SizeClassAllocator64 {
static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; }
static bool conditionVariableEnabled() {
- return ConditionVariableState<typename Config::Primary>::enabled();
+ return Config::hasConditionVariableT();
}
void init(s32 ReleaseToOsInterval) NO_THREAD_SAFETY_ANALYSIS {
@@ -135,7 +138,7 @@ template <typename Config> class SizeClassAllocator64 {
// The actual start of a region is offset by a random number of pages
// when PrimaryEnableRandomOffset is set.
Region->RegionBeg = (PrimaryBase + (I << RegionSizeLog)) +
- (Config::Primary::EnableRandomOffset
+ (Config::getEnableRandomOffset()
? ((getRandomModN(&Seed, 16) + 1) * PageSize)
: 0);
Region->RandState = getRandomU32(&Seed);
@@ -400,9 +403,9 @@ template <typename Config> class SizeClassAllocator64 {
bool setOption(Option O, sptr Value) {
if (O == Option::ReleaseInterval) {
- const s32 Interval = Max(Min(static_cast<s32>(Value),
- Config::Primary::MaxReleaseToOsIntervalMs),
- Config::Primary::MinReleaseToOsIntervalMs);
+ const s32 Interval = Max(
+ Min(static_cast<s32>(Value), Config::getMaxReleaseToOsIntervalMs()),
+ Config::getMinReleaseToOsIntervalMs());
atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval);
return true;
}
@@ -516,7 +519,7 @@ template <typename Config> class SizeClassAllocator64 {
static const uptr NumClasses = SizeClassMap::NumClasses;
static const uptr PrimarySize = RegionSize * NumClasses;
- static const uptr MapSizeIncrement = Config::Primary::MapSizeIncrement;
+ static const uptr MapSizeIncrement = Config::getMapSizeIncrement();
// Fill at most this number of batches from the newly map'd memory.
static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U;
diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h
index c89e6a95f5a68a..d29d3b10536dc5 100644
--- a/compiler-rt/lib/scudo/standalone/secondary.h
+++ b/compiler-rt/lib/scudo/standalone/secondary.h
@@ -151,8 +151,6 @@ template <typename T> class NonZeroLengthArray<T, 0> {
template <typename Config> class MapAllocatorCache {
public:
- using CacheConfig = typename Config::Secondary::Cache;
-
void getStats(ScopedString *Str) {
ScopedLock L(Mutex);
u32 Integral = 0;
@@ -181,16 +179,16 @@ template <typename Config> class MapAllocatorCache {
}
// Ensure the default maximum specified fits the array.
- static_assert(CacheConfig::DefaultMaxEntriesCount <=
- CacheConfig::EntriesArraySize,
+ static_assert(Config::getDefaultMaxEntriesCount() <=
+ Config::getEntriesArraySize(),
"");
void init(s32 ReleaseToOsInterval) NO_THREAD_SAFETY_ANALYSIS {
DCHECK_EQ(EntriesCount, 0U);
setOption(Option::MaxCacheEntriesCount,
- static_cast<sptr>(CacheConfig::DefaultMaxEntriesCount));
+ static_cast<sptr>(Config::getDefaultMaxEntriesCount()));
setOption(Option::MaxCacheEntrySize,
- static_cast<sptr>(CacheConfig::DefaultMaxEntrySize));
+ static_cast<sptr>(Config::getDefaultMaxEntrySize()));
setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
}
@@ -235,9 +233,9 @@ template <typename Config> class MapAllocatorCache {
// just unmap it.
break;
}
- if (CacheConfig::QuarantineSize && useMemoryTagging<Config>(Options)) {
+ if (Config::getQuarantineSize() && useMemoryTagging<Config>(Options)) {
QuarantinePos =
- (QuarantinePos + 1) % Max(CacheConfig::QuarantineSize, 1u);
+ (QuarantinePos + 1) % Max(Config::getQuarantineSize(), 1u);
if (!Quarantine[QuarantinePos].isValid()) {
Quarantine[QuarantinePos] = Entry;
return;
@@ -364,14 +362,14 @@ template <typename Config> class MapAllocatorCache {
bool setOption(Option O, sptr Value) {
if (O == Option::ReleaseInterval) {
const s32 Interval = Max(
- Min(static_cast<s32>(Value), CacheConfig::MaxReleaseToOsIntervalMs),
- CacheConfig::MinReleaseToOsIntervalMs);
+ Min(static_cast<s32>(Value), Config::getMaxReleaseToOsIntervalMs()),
+ Config::getMinReleaseToOsIntervalMs());
atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval);
return true;
}
if (O == Option::MaxCacheEntriesCount) {
const u32 MaxCount = static_cast<u32>(Value);
- if (MaxCount > CacheConfig::EntriesArraySize)
+ if (MaxCount > Config::getEntriesArraySize())
return false;
atomic_store_relaxed(&MaxEntriesCount, MaxCount);
return true;
@@ -388,7 +386,7 @@ template <typename Config> class MapAllocatorCache {
void disableMemoryTagging() EXCLUDES(Mutex) {
ScopedLock L(Mutex);
- for (u32 I = 0; I != CacheConfig::QuarantineSize; ++I) {
+ for (u32 I = 0; I != Config::getQuarantineSize(); ++I) {
if (Quarantine[I].isValid()) {
MemMapT &MemMap = Quarantine[I].MemMap;
MemMap.unmap(MemMap.getBase(), MemMap.getCapacity());
@@ -413,11 +411,11 @@ template <typename Config> class MapAllocatorCache {
private:
void empty() {
- MemMapT MapInfo[CacheConfig::EntriesArraySize];
+ MemMapT MapInfo[Config::getEntriesArraySize()];
uptr N = 0;
{
ScopedLock L(Mutex);
- for (uptr I = 0; I < CacheConfig::EntriesArraySize; I++) {
+ for (uptr I = 0; I < Config::getEntriesArraySize(); I++) {
if (!Entries[I].isValid())
continue;
MapInfo[N] = Entries[I].MemMap;
@@ -450,9 +448,9 @@ template <typename Config> class MapAllocatorCache {
if (!EntriesCount || OldestTime == 0 || OldestTime > Time)
return;
OldestTime = 0;
- for (uptr I = 0; I < CacheConfig::QuarantineSize; I++)
+ for (uptr I = 0; I < Config::getQuarantineSize(); I++)
releaseIfOlderThan(Quarantine[I], Time);
- for (uptr I = 0; I < CacheConfig::EntriesArraySize; I++)
+ for (uptr I = 0; I < Config::getEntriesArraySize(); I++)
releaseIfOlderThan(Entries[I], Time);
}
@@ -467,8 +465,8 @@ template <typename Config> class MapAllocatorCache {
u32 CallsToRetrieve GUARDED_BY(Mutex) = 0;
u32 SuccessfulRetrieves GUARDED_BY(Mutex) = 0;
- CachedBlock Entries[CacheConfig::EntriesArraySize] GUARDED_BY(Mutex) = {};
- NonZeroLengthArray<CachedBlock, CacheConfig::QuarantineSize>
+ CachedBlock Entries[Config::getEntriesArraySize()] GUARDED_BY(Mutex) = {};
+ NonZeroLengthArray<CachedBlock, Config::getQuarantineSize()>
Quarantine GUARDED_BY(Mutex) = {};
};
@@ -537,7 +535,7 @@ template <typename Config> class MapAllocator {
void getStats(ScopedString *Str);
private:
- typename Config::Secondary::template CacheT<Config> Cache;
+ typename Config::template CacheT<typename Config::CacheConfig> Cache;
mutable HybridMutex Mutex;
DoublyLinkedList<LargeBlock::Header> InUseBlocks GUARDED_BY(Mutex);
diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
index 3dbd93cacefd68..46f17c642d0acd 100644
--- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
@@ -189,7 +189,6 @@ struct TestConditionVariableConfig {
#endif
static const scudo::s32 MinReleaseToOsIntervalMs = 1000;
static const scudo::s32 MaxReleaseToOsIntervalMs = 1000;
- static const bool UseConditionVariable = true;
#if SCUDO_LINUX
using ConditionVariableT = scudo::ConditionVariableLinux;
#else
diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
index f64a5143b30d46..683ce3e596596d 100644
--- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
@@ -9,6 +9,7 @@
#include "tests/scudo_unit_test.h"
#include "allocator_config.h"
+#include "allocator_config_wrapper.h"
#include "condition_variable.h"
#include "primary32.h"
#include "primary64.h"
@@ -29,6 +30,9 @@
template <typename SizeClassMapT> struct TestConfig1 {
static const bool MaySupportMemoryTagging = false;
+ template <typename> using TSDRegistryT = void;
+ template <typename> using PrimaryT = void;
+ template <typename> using SecondaryT = void;
struct Primary {
using SizeClassMap = SizeClassMapT;
@@ -45,6 +49,9 @@ template <typename SizeClassMapT> struct TestConfig1 {
template <typename SizeClassMapT> struct TestConfig2 {
static const bool MaySupportMemoryTagging = false;
+ template <typename> using TSDRegistryT = void;
+ template <typename> using PrimaryT = void;
+ template <typename> using SecondaryT = void;
struct Primary {
using SizeClassMap = SizeClassMapT;
@@ -66,6 +73,9 @@ template <typename SizeClassMapT> struct TestConfig2 {
template <typename SizeClassMapT> struct TestConfig3 {
static const bool MaySupportMemoryTagging = true;
+ template <typename> using TSDRegistryT = void;
+ template <typename> using PrimaryT = void;
+ template <typename> using SecondaryT = void;
struct Primary {
using SizeClassMap = SizeClassMapT;
@@ -87,6 +97,9 @@ template <typename SizeClassMapT> struct TestConfig3 {
template <typename SizeClassMapT> struct TestConfig4 {
static const bool MaySupportMemoryTagging = true;
+ template <typename> using TSDRegistryT = void;
+ template <typename> using PrimaryT = void;
+ template <typename> using SecondaryT = void;
struct Primary {
using SizeClassMap = SizeClassMapT;
@@ -109,6 +122,9 @@ template <typename SizeClassMapT> struct TestConfig4 {
// This is the only test config that enables the condition variable.
template <typename SizeClassMapT> struct TestConfig5 {
static const bool MaySupportMemoryTagging = true;
+ template <typename> using TSDRegistryT = void;
+ template <typename> using PrimaryT = void;
+ template <typename> using SecondaryT = void;
struct Primary {
using SizeClassMap = SizeClassMapT;
@@ -125,7 +141,6 @@ template <typename SizeClassMapT> struct TestConfig5 {
typedef scudo::u32 CompactPtrT;
static const bool EnableRandomOffset = true;
static const scudo::uptr MapSizeIncrement = 1UL << 18;
- static const bool UseConditionVariable = true;
#if SCUDO_LINUX
using ConditionVariableT = scudo::ConditionVariableLinux;
#else
@@ -139,10 +154,12 @@ struct Config : public BaseConfig<SizeClassMapT> {};
template <template <typename> class BaseConfig, typename SizeClassMapT>
struct SizeClassAllocator
- : public scudo::SizeClassAllocator64<Config<BaseConfig, SizeClassMapT>> {};
+ : public scudo::SizeClassAllocator64<
+ scudo::PrimaryConfig<Config<BaseConfig, SizeClassMapT>>> {};
template <typename SizeClassMapT>
struct SizeClassAllocator<TestConfig1, SizeClassMapT>
- : public scudo::SizeClassAllocator32<Config<TestConfig1, SizeClassMapT>> {};
+ : public scudo::SizeClassAllocator32<
+ scudo::PrimaryConfig<Config<TestConfig1, SizeClassMapT>>> {};
template <template <typename> class BaseConfig, typename SizeClassMapT>
struct TestAllocator : public SizeClassAllocator<BaseConfig, SizeClassMapT> {
@@ -219,6 +236,9 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, BasicPrimary) {
struct SmallRegionsConfig {
static const bool MaySupportMemoryTagging = false;
+ template <typename> using TSDRegistryT = void;
+ template <typename> using PrimaryT = void;
+ template <typename> using SecondaryT = void;
struct Primary {
using SizeClassMap = scudo::DefaultSizeClassMap;
@@ -236,7 +256,8 @@ struct SmallRegionsConfig {
// The 64-bit SizeClassAllocator can be easily OOM'd with small region sizes.
// For the 32-bit one, it requires actually exhausting memory, so we skip it.
TEST(ScudoPrimaryTest, Primary64OOM) {
- using Primary = scudo::SizeClassAllocator64<SmallRegionsConfig>;
+ using Primary =
+ scudo::SizeClassAllocator64<scudo::PrimaryConfig<SmallRegionsConfig>>;
Primary Allocator;
Allocator.init(/*ReleaseToOsInterval=*/-1);
typename Primary::CacheT Cache;
diff --git a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
index 18d2e187fa3ce2..8f0250e88ebf3a 100644
--- a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
@@ -10,6 +10,7 @@
#include "tests/scudo_unit_test.h"
#include "allocator_config.h"
+#include "allocator_config_wrapper.h"
#include "secondary.h"
#include <algorithm>
@@ -22,7 +23,8 @@
#include <vector>
template <typename Config> static scudo::Options getOptionsForConfig() {
- if (!Config::MaySupportMemoryTagging || !scudo::archSupportsMemoryTagging() ||
+ if (!Config::getMaySupportMemoryTagging() ||
+ !scudo::archSupportsMemoryTagging() ||
!scudo::systemSupportsMemoryTagging())
return {};
scudo::AtomicOptions AO;
@@ -31,8 +33,9 @@ template <typename Config> static scudo::Options getOptionsForConfig() {
}
template <typename Config> static void testSecondaryBasic(void) {
- using SecondaryT = scudo::MapAllocator<Config>;
- scudo::Options Options = getOptionsForConfig<Config>();
+ using SecondaryT = scudo::MapAllocator<scudo::SecondaryConfig<Config>>;
+ scudo::Options Options =
+ getOptionsForConfig<scudo::SecondaryConfig<Config>>();
scudo::GlobalStats S;
S.init();
@@ -84,6 +87,10 @@ template <typename Config> static void testSecondaryBasic(void) {
struct NoCacheConfig {
static const bool MaySupportMemoryTagging = false;
+ template <typename> using TSDRegistryT = void;
+ template <typename> using PrimaryT = void;
+ template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
+
struct Secondary {
template <typename Config>
using CacheT = scudo::MapAllocatorNoCache<Config>;
@@ -92,6 +99,10 @@ struct NoCacheConfig {
struct TestConfig {
static const bool MaySupportMemoryTagging = false;
+ template <typename> using TSDRegistryT = void;
+ template <typename> using PrimaryT = void;
+ template <typename> using SecondaryT = void;
+
struct Secondary {
struct Cache {
static const scudo::u32 EntriesArraySize = 128U;
@@ -114,7 +125,7 @@ TEST(ScudoSecondaryTest, SecondaryBasic) {
struct MapAllocatorTest : public Test {
using Config = scudo::DefaultConfig;
- using LargeAllocator = scudo::MapAllocator<Config>;
+ using LargeAllocator = scudo::MapAllocator<scudo::SecondaryConfig<Config>>;
void SetUp() override { Allocator->init(nullptr); }
@@ -122,7 +133,8 @@ struct MapAllocatorTest : public Test {
std::unique_ptr<LargeAllocator> Allocator =
std::make_unique<LargeAllocator>();
- scudo::Options Options = getOptionsForConfig<Config>();
+ scudo::Options Options =
+ getOptionsForConfig<scudo::SecondaryConfig<Config>>();
};
// This exercises a variety of combinations of size and alignment for the
>From e157950c18e0e9aa6328f2ebf6463a9c7fbfc01e Mon Sep 17 00:00:00 2001
From: Chia-hung Duan <chiahungduan at google.com>
Date: Mon, 26 Feb 2024 19:32:38 +0000
Subject: [PATCH 5/5] [scudo] Support no-preserve-all-regions mode
This releases the requirement that we need to preserve the memory for
all regions at the beginning. It needs a huge amount of contiguous pages
and which may be a challenge in certain cases. Therefore, adding a new
flag, PreserveAllRegions, to indicate whether we want to allocate the
regions on demand.
Note that once the PreserveAllRegions is enabled, EnableRandomOffset
becomes irrelevant because the base is already random.
---
.../lib/scudo/standalone/allocator_config.def | 8 +-
compiler-rt/lib/scudo/standalone/primary64.h | 128 +++++++++++-------
.../scudo/standalone/tests/primary_test.cpp | 1 +
3 files changed, 89 insertions(+), 48 deletions(-)
diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.def b/compiler-rt/lib/scudo/standalone/allocator_config.def
index ddde71d2463f78..1ca4fcffd0a272 100644
--- a/compiler-rt/lib/scudo/standalone/allocator_config.def
+++ b/compiler-rt/lib/scudo/standalone/allocator_config.def
@@ -86,9 +86,15 @@ PRIMARY_REQUIRED(const s32, MaxReleaseToOsIntervalMs)
// PRIMARY_OPTIONAL(TYPE, NAME, DEFAULT)
//
// Indicates support for offsetting the start of a region by a random number of
-// pages. Only used with primary64.
+// pages. This is only used if `PreserveAllRegions` is enabled.
PRIMARY_OPTIONAL(const bool, EnableRandomOffset, false)
+// When `PreserveAllRegions` is true, the virtual address for all regions will
+// be preserved within a big chunk of memory. This will reduce the fragmentation
+// caused by region allocations but may require a huge amount of contiguous
+// pages at initialization.
+PRIMARY_OPTIONAL(const bool, PreserveAllRegions, true)
+
// PRIMARY_OPTIONAL_TYPE(NAME, DEFAULT)
//
// Use condition variable to shorten the waiting time of refillment of
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index c53e8f98547a18..6ac4f3c1e17bb0 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -121,40 +121,24 @@ template <typename Config> class SizeClassAllocator64 {
SmallerBlockReleasePageDelta =
PagesInGroup * (1 + MinSizeClass / 16U) / 100;
- // Reserve the space required for the Primary.
- CHECK(ReservedMemory.create(/*Addr=*/0U, PrimarySize,
- "scudo:primary_reserve"));
- PrimaryBase = ReservedMemory.getBase();
- DCHECK_NE(PrimaryBase, 0U);
-
- u32 Seed;
- const u64 Time = getMonotonicTimeFast();
- if (!getRandom(reinterpret_cast<void *>(&Seed), sizeof(Seed)))
- Seed = static_cast<u32>(Time ^ (PrimaryBase >> 12));
-
- for (uptr I = 0; I < NumClasses; I++) {
- RegionInfo *Region = getRegionInfo(I);
-
- // The actual start of a region is offset by a random number of pages
- // when PrimaryEnableRandomOffset is set.
- Region->RegionBeg = (PrimaryBase + (I << RegionSizeLog)) +
- (Config::getEnableRandomOffset()
- ? ((getRandomModN(&Seed, 16) + 1) * PageSize)
- : 0);
- Region->RandState = getRandomU32(&Seed);
- // Releasing small blocks is expensive, set a higher threshold to avoid
- // frequent page releases.
- if (isSmallBlock(getSizeByClassId(I)))
- Region->TryReleaseThreshold = PageSize * SmallerBlockReleasePageDelta;
- else
- Region->TryReleaseThreshold = PageSize;
- Region->ReleaseInfo.LastReleaseAtNs = Time;
-
- Region->MemMapInfo.MemMap = ReservedMemory.dispatch(
- PrimaryBase + (I << RegionSizeLog), RegionSize);
- CHECK(Region->MemMapInfo.MemMap.isAllocated());
+ if (Config::getPreserveAllRegions()) {
+ ReservedMemoryT ReservedMemory = {};
+ // Reserve the space required for the Primary.
+ CHECK(ReservedMemory.create(/*Addr=*/0U, RegionSize * NumClasses,
+ "scudo:primary_reserve"));
+ const uptr PrimaryBase = ReservedMemory.getBase();
+ const bool EnableRandomOffset =
+ Config::getPreserveAllRegions() && Config::getEnableRandomOffset();
+
+ for (uptr I = 0; I < NumClasses; I++) {
+ MemMapT RegionMemMap = ReservedMemory.dispatch(
+ PrimaryBase + (I << RegionSizeLog), RegionSize);
+ RegionInfo *Region = getRegionInfo(I);
+
+ initRegion(Region, I, RegionMemMap, EnableRandomOffset);
+ }
+ shuffle(RegionInfoArray, NumClasses, &getRegionInfo(0)->RandState);
}
- shuffle(RegionInfoArray, NumClasses, &Seed);
// The binding should be done after region shuffling so that it won't bind
// the FLLock from the wrong region.
@@ -164,14 +148,17 @@ template <typename Config> class SizeClassAllocator64 {
setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
}
- void unmapTestOnly() NO_THREAD_SAFETY_ANALYSIS {
+ void unmapTestOnly() {
for (uptr I = 0; I < NumClasses; I++) {
RegionInfo *Region = getRegionInfo(I);
+ {
+ ScopedLock ML(Region->MMLock);
+ MemMapT MemMap = Region->MemMapInfo.MemMap;
+ if (MemMap.isAllocated())
+ MemMap.unmap(MemMap.getBase(), MemMap.getCapacity());
+ }
*Region = {};
}
- if (PrimaryBase)
- ReservedMemory.release();
- PrimaryBase = 0U;
}
// When all blocks are freed, it has to be the same size as `AllocatedUser`.
@@ -255,9 +242,10 @@ template <typename Config> class SizeClassAllocator64 {
}
const bool RegionIsExhausted = Region->Exhausted;
- if (!RegionIsExhausted)
+ if (!RegionIsExhausted) {
PopCount = populateFreeListAndPopBlocks(C, ClassId, Region, ToArray,
MaxBlockCount);
+ }
ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
break;
}
@@ -517,7 +505,6 @@ template <typename Config> class SizeClassAllocator64 {
private:
static const uptr RegionSize = 1UL << RegionSizeLog;
static const uptr NumClasses = SizeClassMap::NumClasses;
- static const uptr PrimarySize = RegionSize * NumClasses;
static const uptr MapSizeIncrement = Config::getMapSizeIncrement();
// Fill at most this number of batches from the newly map'd memory.
@@ -573,9 +560,14 @@ template <typename Config> class SizeClassAllocator64 {
}
uptr getRegionBaseByClassId(uptr ClassId) {
- return roundDown(getRegionInfo(ClassId)->RegionBeg - PrimaryBase,
- RegionSize) +
- PrimaryBase;
+ RegionInfo *Region = getRegionInfo(ClassId);
+ Region->MMLock.assertHeld();
+
+ if (!Config::getPreserveAllRegions() &&
+ !Region->MemMapInfo.MemMap.isAllocated()) {
+ return 0U;
+ }
+ return Region->MemMapInfo.MemMap.getBase();
}
static CompactPtrT compactPtrInternal(uptr Base, uptr Ptr) {
@@ -605,6 +597,35 @@ template <typename Config> class SizeClassAllocator64 {
return BlockSize > PageSize;
}
+ void initRegion(RegionInfo *Region, uptr ClassId, MemMapT MemMap,
+ bool EnableRandomOffset) REQUIRES(Region->MMLock) {
+ DCHECK(!Region->MemMapInfo.MemMap.isAllocated());
+ DCHECK(MemMap.isAllocated());
+
+ const uptr PageSize = getPageSizeCached();
+ const uptr RegionBase = MemMap.getBase();
+
+ Region->MemMapInfo.MemMap = MemMap;
+
+ u32 Seed;
+ const u64 Time = getMonotonicTimeFast();
+ if (!getRandom(reinterpret_cast<void *>(&Seed), sizeof(Seed)))
+ Seed = static_cast<u32>(Time ^ (RegionBase >> 12));
+
+ Region->RegionBeg = RegionBase;
+ if (EnableRandomOffset)
+ Region->RegionBeg += (getRandomModN(&Seed, 16) + 1) * PageSize;
+
+ // Releasing small blocks is expensive, set a higher threshold to avoid
+ // frequent page releases.
+ if (isSmallBlock(getSizeByClassId(ClassId)))
+ Region->TryReleaseThreshold = PageSize * SmallerBlockReleasePageDelta;
+ else
+ Region->TryReleaseThreshold = PageSize;
+
+ Region->ReleaseInfo.LastReleaseAtNs = Time;
+ }
+
void pushBatchClassBlocks(RegionInfo *Region, CompactPtrT *Array, u32 Size)
REQUIRES(Region->FLLock) {
DCHECK_EQ(Region, getRegionInfo(SizeClassMap::BatchClassId));
@@ -992,9 +1013,26 @@ template <typename Config> class SizeClassAllocator64 {
CompactPtrT *ToArray,
const u16 MaxBlockCount)
REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
+ if (!Config::getPreserveAllRegions() &&
+ !Region->MemMapInfo.MemMap.isAllocated()) {
+ ReservedMemoryT ReservedMemory;
+ if (UNLIKELY(!ReservedMemory.create(/*Addr=*/0U, RegionSize,
+ "scudo:primary_reserve",
+ MAP_ALLOWNOMEM))) {
+ Printf("Can't preserve pages for size class %zu.\n",
+ getSizeByClassId(ClassId));
+ Region->Exhausted = true;
+ return 0U;
+ }
+ initRegion(Region, ClassId,
+ ReservedMemory.dispatch(ReservedMemory.getBase(),
+ ReservedMemory.getCapacity()),
+ /*EnableRandomOffset*/ false);
+ }
+
+ DCHECK(Region->MemMapInfo.MemMap.isAllocated());
const uptr Size = getSizeByClassId(ClassId);
const u16 MaxCount = CacheT::getMaxCached(Size);
-
const uptr RegionBeg = Region->RegionBeg;
const uptr MappedUser = Region->MemMapInfo.MappedUser;
const uptr TotalUserBytes =
@@ -1682,10 +1720,6 @@ template <typename Config> class SizeClassAllocator64 {
Region->FLLockCV.notifyAll(Region->FLLock);
}
- // TODO: `PrimaryBase` can be obtained from ReservedMemory. This needs to be
- // deprecated.
- uptr PrimaryBase = 0;
- ReservedMemoryT ReservedMemory = {};
// The minimum size of pushed blocks that we will try to release the pages in
// that size class.
uptr SmallerBlockReleasePageDelta = 0;
diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
index 683ce3e596596d..a6bbf98026455e 100644
--- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
@@ -90,6 +90,7 @@ template <typename SizeClassMapT> struct TestConfig3 {
static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX;
typedef scudo::uptr CompactPtrT;
static const scudo::uptr CompactPtrScale = 0;
+ static const bool PreserveAllRegions = false;
static const bool EnableRandomOffset = true;
static const scudo::uptr MapSizeIncrement = 1UL << 18;
};
More information about the llvm-commits
mailing list