[compiler-rt] 0fb2aee - Use u16 to store Count/MaxCount
Chia-hung Duan via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 13 16:35:18 PDT 2022
Author: Chia-hung Duan
Date: 2022-10-13T23:35:06Z
New Revision: 0fb2aeef5310eaba2915b30810464a744a80da15
URL: https://github.com/llvm/llvm-project/commit/0fb2aeef5310eaba2915b30810464a744a80da15
DIFF: https://github.com/llvm/llvm-project/commit/0fb2aeef5310eaba2915b30810464a744a80da15.diff
LOG: Use u16 to store Count/MaxCount
The Count/MaxCount used in TransferBatch and PerClass can be fit in u16 in
current configurations and it's also reasonable to have a u16 limit. The
spare 16 bits will be used for additional status like pages mapping
status in a TransferBatch.
Reviewed By: cryptoad, cferris, vitalybuka
Differential Revision: https://reviews.llvm.org/D133145
Added:
Modified:
compiler-rt/lib/scudo/standalone/local_cache.h
compiler-rt/lib/scudo/standalone/primary32.h
compiler-rt/lib/scudo/standalone/primary64.h
compiler-rt/lib/scudo/standalone/release.h
compiler-rt/lib/scudo/standalone/size_class_map.h
compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
compiler-rt/lib/scudo/standalone/tests/release_test.cpp
compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp
compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp
Removed:
################################################################################
diff --git a/compiler-rt/lib/scudo/standalone/local_cache.h b/compiler-rt/lib/scudo/standalone/local_cache.h
index f46645f9badf2..dfdea00529c7d 100644
--- a/compiler-rt/lib/scudo/standalone/local_cache.h
+++ b/compiler-rt/lib/scudo/standalone/local_cache.h
@@ -10,6 +10,7 @@
#define SCUDO_LOCAL_CACHE_H_
#include "internal_defs.h"
+#include "platform.h"
#include "report.h"
#include "stats.h"
@@ -20,8 +21,8 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
typedef typename SizeClassAllocator::CompactPtrT CompactPtrT;
struct TransferBatch {
- static const u32 MaxNumCached = SizeClassMap::MaxNumCachedHint;
- void setFromArray(CompactPtrT *Array, u32 N) {
+ static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint;
+ void setFromArray(CompactPtrT *Array, u16 N) {
DCHECK_LE(N, MaxNumCached);
Count = N;
memcpy(Batch, Array, sizeof(Batch[0]) * Count);
@@ -34,19 +35,19 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
void copyToArray(CompactPtrT *Array) const {
memcpy(Array, Batch, sizeof(Batch[0]) * Count);
}
- u32 getCount() const { return Count; }
- CompactPtrT get(u32 I) const {
+ u16 getCount() const { return Count; }
+ CompactPtrT get(u16 I) const {
DCHECK_LE(I, Count);
return Batch[I];
}
- static u32 getMaxCached(uptr Size) {
+ static u16 getMaxCached(uptr Size) {
return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size));
}
TransferBatch *Next;
private:
- u32 Count;
CompactPtrT Batch[MaxNumCached];
+ u16 Count;
};
void init(GlobalStats *S, SizeClassAllocator *A) {
@@ -128,9 +129,9 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
private:
static const uptr NumClasses = SizeClassMap::NumClasses;
static const uptr BatchClassId = SizeClassMap::BatchClassId;
- struct PerClass {
- u32 Count;
- u32 MaxCount;
+ struct alignas(SCUDO_CACHE_LINE_SIZE) PerClass {
+ u16 Count;
+ u16 MaxCount;
// Note: ClassSize is zero for the transfer batch.
uptr ClassSize;
CompactPtrT Chunks[2 * TransferBatch::MaxNumCached];
@@ -180,7 +181,7 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
}
NOINLINE void drain(PerClass *C, uptr ClassId) {
- const u32 Count = Min(C->MaxCount / 2, C->Count);
+ const u16 Count = Min(static_cast<u16>(C->MaxCount / 2), C->Count);
TransferBatch *B =
createBatch(ClassId, Allocator->decompactPtr(ClassId, C->Chunks[0]));
if (UNLIKELY(!B))
diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index 326c10a32a852..9bc97ec539822 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -351,7 +351,7 @@ template <typename Config> class SizeClassAllocator32 {
}
const uptr Size = getSizeByClassId(ClassId);
- const u32 MaxCount = TransferBatch::getMaxCached(Size);
+ const u16 MaxCount = TransferBatch::getMaxCached(Size);
DCHECK_GT(MaxCount, 0U);
// The maximum number of blocks we should carve in the region is dictated
// by the maximum number of batches we want to fill, and the amount of
@@ -382,7 +382,8 @@ template <typename Config> class SizeClassAllocator32 {
C->createBatch(ClassId, reinterpret_cast<void *>(ShuffleArray[I]));
if (UNLIKELY(!B))
return nullptr;
- const u32 N = Min(MaxCount, NumberOfBlocks - I);
+ // `MaxCount` is u16 so the result will also fit in u16.
+ const u16 N = static_cast<u16>(Min<u32>(MaxCount, NumberOfBlocks - I));
B->setFromArray(&ShuffleArray[I], N);
Sci->FreeList.push_back(B);
I += N;
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 14784ee8f3772..e03c2b56a9b79 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -333,7 +333,7 @@ template <typename Config> class SizeClassAllocator64 {
NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId,
RegionInfo *Region) {
const uptr Size = getSizeByClassId(ClassId);
- const u32 MaxCount = TransferBatch::getMaxCached(Size);
+ const u16 MaxCount = TransferBatch::getMaxCached(Size);
const uptr RegionBeg = Region->RegionBeg;
const uptr MappedUser = Region->MappedUser;
@@ -392,7 +392,8 @@ template <typename Config> class SizeClassAllocator64 {
CompactPtrBase, ShuffleArray[I])));
if (UNLIKELY(!B))
return nullptr;
- const u32 N = Min(MaxCount, NumberOfBlocks - I);
+ // `MaxCount` is u16 so the result will also fit in u16.
+ const u16 N = static_cast<u16>(Min<u32>(MaxCount, NumberOfBlocks - I));
B->setFromArray(&ShuffleArray[I], N);
Region->FreeList.push_back(B);
I += N;
diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h
index 49cc6ae618af7..fdb62b4412643 100644
--- a/compiler-rt/lib/scudo/standalone/release.h
+++ b/compiler-rt/lib/scudo/standalone/release.h
@@ -242,7 +242,7 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
if (BlockSize <= PageSize && PageSize % BlockSize == 0) {
// Each chunk affects one page only.
for (const auto &It : FreeList) {
- for (u32 I = 0; I < It.getCount(); I++) {
+ for (u16 I = 0; I < It.getCount(); I++) {
const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
if (P >= RoundedSize)
continue;
@@ -256,7 +256,7 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
DCHECK_GE(RegionSize, BlockSize);
const uptr LastBlockInRegion = ((RegionSize / BlockSize) - 1U) * BlockSize;
for (const auto &It : FreeList) {
- for (u32 I = 0; I < It.getCount(); I++) {
+ for (u16 I = 0; I < It.getCount(); I++) {
const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
if (P >= RoundedSize)
continue;
diff --git a/compiler-rt/lib/scudo/standalone/size_class_map.h b/compiler-rt/lib/scudo/standalone/size_class_map.h
index 8d5a560dc5ba3..cc369ff5797fa 100644
--- a/compiler-rt/lib/scudo/standalone/size_class_map.h
+++ b/compiler-rt/lib/scudo/standalone/size_class_map.h
@@ -23,7 +23,7 @@ inline uptr scaledLog2(uptr Size, uptr ZeroLog, uptr LogBits) {
}
template <typename Config> struct SizeClassMapBase {
- static u32 getMaxCachedHint(uptr Size) {
+ static u16 getMaxCachedHint(uptr Size) {
DCHECK_NE(Size, 0);
u32 N;
// Force a 32-bit division if the template parameters allow for it.
@@ -31,7 +31,10 @@ template <typename Config> struct SizeClassMapBase {
N = static_cast<u32>((1UL << Config::MaxBytesCachedLog) / Size);
else
N = (1U << Config::MaxBytesCachedLog) / static_cast<u32>(Size);
- return Max(1U, Min(Config::MaxNumCachedHint, N));
+
+ // Note that Config::MaxNumCachedHint is u16 so the result is guaranteed to
+ // fit in u16.
+ return static_cast<u16>(Max(1U, Min<u32>(Config::MaxNumCachedHint, N)));
}
};
@@ -65,7 +68,7 @@ class FixedSizeClassMap : public SizeClassMapBase<Config> {
static const uptr M = (1UL << S) - 1;
public:
- static const u32 MaxNumCachedHint = Config::MaxNumCachedHint;
+ static const u16 MaxNumCachedHint = Config::MaxNumCachedHint;
static const uptr MaxSize = (1UL << Config::MaxSizeLog) + Config::SizeDelta;
static const uptr NumClasses =
@@ -99,7 +102,7 @@ class FixedSizeClassMap : public SizeClassMapBase<Config> {
return MidClass + 1 + scaledLog2(Size - 1, Config::MidSizeLog, S);
}
- static u32 getMaxCachedHint(uptr Size) {
+ static u16 getMaxCachedHint(uptr Size) {
DCHECK_LE(Size, MaxSize);
return Base::getMaxCachedHint(Size);
}
@@ -178,7 +181,7 @@ class TableSizeClassMap : public SizeClassMapBase<Config> {
static constexpr LSBTable LTable = {};
public:
- static const u32 MaxNumCachedHint = Config::MaxNumCachedHint;
+ static const u16 MaxNumCachedHint = Config::MaxNumCachedHint;
static const uptr NumClasses = ClassesSize + 1;
static_assert(NumClasses < 256, "");
@@ -212,7 +215,7 @@ class TableSizeClassMap : public SizeClassMapBase<Config> {
return SzTable.Tab[scaledLog2(Size - 1, Config::MidSizeLog, S)];
}
- static u32 getMaxCachedHint(uptr Size) {
+ static u16 getMaxCachedHint(uptr Size) {
DCHECK_LE(Size, MaxSize);
return Base::getMaxCachedHint(Size);
}
@@ -223,7 +226,7 @@ struct DefaultSizeClassConfig {
static const uptr MinSizeLog = 5;
static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 17;
- static const u32 MaxNumCachedHint = 14;
+ static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = 0;
};
@@ -235,7 +238,7 @@ struct FuchsiaSizeClassConfig {
static const uptr MinSizeLog = 5;
static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 17;
- static const u32 MaxNumCachedHint = 12;
+ static const u16 MaxNumCachedHint = 12;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize();
};
@@ -248,7 +251,7 @@ struct AndroidSizeClassConfig {
static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 6;
static const uptr MaxSizeLog = 16;
- static const u32 MaxNumCachedHint = 13;
+ static const u16 MaxNumCachedHint = 13;
static const uptr MaxBytesCachedLog = 13;
static constexpr u32 Classes[] = {
@@ -263,7 +266,7 @@ struct AndroidSizeClassConfig {
static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 16;
- static const u32 MaxNumCachedHint = 14;
+ static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 13;
static constexpr u32 Classes[] = {
@@ -292,7 +295,7 @@ struct SvelteSizeClassConfig {
static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 14;
- static const u32 MaxNumCachedHint = 13;
+ static const u16 MaxNumCachedHint = 13;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize();
#else
@@ -300,7 +303,7 @@ struct SvelteSizeClassConfig {
static const uptr MinSizeLog = 3;
static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 14;
- static const u32 MaxNumCachedHint = 14;
+ static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize();
#endif
@@ -315,7 +318,7 @@ struct TrustySizeClassConfig {
static const uptr MinSizeLog = 7;
static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 7;
- static const u32 MaxNumCachedHint = 8;
+ static const u16 MaxNumCachedHint = 8;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = 0;
};
diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
index 94d97df816771..49c1ba9f520c0 100644
--- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
@@ -506,7 +506,7 @@ struct DeathSizeClassConfig {
static const scudo::uptr MinSizeLog = 10;
static const scudo::uptr MidSizeLog = 10;
static const scudo::uptr MaxSizeLog = 13;
- static const scudo::u32 MaxNumCachedHint = 4;
+ static const scudo::u16 MaxNumCachedHint = 4;
static const scudo::uptr MaxBytesCachedLog = 12;
static const scudo::uptr SizeDelta = 0;
};
diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
index 283e2973c1e62..d70dcc9e3db37 100644
--- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
@@ -176,7 +176,7 @@ TEST(ScudoPrimaryTest, Primary64OOM) {
AllocationFailed = true;
break;
}
- for (scudo::u32 J = 0; J < B->getCount(); J++)
+ for (scudo::u16 J = 0; J < B->getCount(); J++)
memset(Allocator.decompactPtr(ClassId, B->get(J)), 'B', Size);
Batches.push_back(B);
}
diff --git a/compiler-rt/lib/scudo/standalone/tests/release_test.cpp b/compiler-rt/lib/scudo/standalone/tests/release_test.cpp
index 04c02891e9156..3b62b639e46b2 100644
--- a/compiler-rt/lib/scudo/standalone/tests/release_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/release_test.cpp
@@ -130,22 +130,22 @@ class ReleasedPagesRecorder {
// Simplified version of a TransferBatch.
template <class SizeClassMap> struct FreeBatch {
- static const scudo::u32 MaxCount = SizeClassMap::MaxNumCachedHint;
+ static const scudo::u16 MaxCount = SizeClassMap::MaxNumCachedHint;
void clear() { Count = 0; }
void add(scudo::uptr P) {
DCHECK_LT(Count, MaxCount);
Batch[Count++] = P;
}
- scudo::u32 getCount() const { return Count; }
- scudo::uptr get(scudo::u32 I) const {
+ scudo::u16 getCount() const { return Count; }
+ scudo::uptr get(scudo::u16 I) const {
DCHECK_LE(I, Count);
return Batch[I];
}
FreeBatch *Next;
private:
- scudo::u32 Count;
scudo::uptr Batch[MaxCount];
+ scudo::u16 Count;
};
template <class SizeClassMap> void testReleaseFreeMemoryToOS() {
diff --git a/compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp b/compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp
index 076f36f86be44..b11db1e9f645e 100644
--- a/compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp
@@ -33,7 +33,7 @@ struct OneClassSizeClassConfig {
static const scudo::uptr MinSizeLog = 5;
static const scudo::uptr MidSizeLog = 5;
static const scudo::uptr MaxSizeLog = 5;
- static const scudo::u32 MaxNumCachedHint = 0;
+ static const scudo::u16 MaxNumCachedHint = 0;
static const scudo::uptr MaxBytesCachedLog = 0;
static const scudo::uptr SizeDelta = 0;
};
@@ -48,7 +48,7 @@ struct LargeMaxSizeClassConfig {
static const scudo::uptr MinSizeLog = 4;
static const scudo::uptr MidSizeLog = 8;
static const scudo::uptr MaxSizeLog = 63;
- static const scudo::u32 MaxNumCachedHint = 128;
+ static const scudo::u16 MaxNumCachedHint = 128;
static const scudo::uptr MaxBytesCachedLog = 16;
static const scudo::uptr SizeDelta = 0;
};
diff --git a/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp b/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp
index 8b17be0e965b3..bcaa5834932ca 100644
--- a/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp
+++ b/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp
@@ -140,7 +140,7 @@ struct MySizeClassConfig {
static const uptr MinSizeLog = %zu;
static const uptr MidSizeLog = %zu;
static const uptr MaxSizeLog = %zu;
- static const u32 MaxNumCachedHint = 14;
+ static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 14;
static constexpr u32 Classes[] = {)",
More information about the llvm-commits
mailing list