[llvm] [Allocator] Keep bump pointer at a minimum alignment (PR #203718)
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Sun Jun 14 15:33:00 PDT 2026
https://github.com/MaskRay updated https://github.com/llvm/llvm-project/pull/203718
>From eec98ef424ed8b6f2fa125e523070d966c6583c6 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Fri, 5 Jun 2026 23:53:22 -0700
Subject: [PATCH 1/3] [Allocator] Keep bump pointer at a minimum alignment
Add a `MinAlign` template parameter (default 8, sizeof(size_t) on 64-bit
platforms) so that the common case `Alignment <= MinAlign` can skip
realigning `CurPtr`.
This is achieved by rounding each allocation's size up to MinAlign, so
the bump pointer stays MinAlign-aligned between allocations.
SpecificBumpPtrAllocator::DestroyAll() walks objects at a fixed sizeof(T)
stride and needs tight packing, so it uses MinAlign=1.
Adjust the fast path check to avoid a separate null check for an emnpty
allocator (`End == nullptr`).
Also use `bit_ceil` instead of `NextPowerOf2` in the placement operator
new alignment heuristic. NextPowerOf2 is strictly greater, so an 8-byte
pointer-sized object asked for alignment 16 and missed the fast path;
bit_ceil(8) == 8 keeps it on the fast path.
---
llvm/include/llvm/Support/Allocator.h | 67 ++++++++++++++++-----------
1 file changed, 41 insertions(+), 26 deletions(-)
diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h
index fffcbd9f3c1d8..eae99d7f8e773 100644
--- a/llvm/include/llvm/Support/Allocator.h
+++ b/llvm/include/llvm/Support/Allocator.h
@@ -59,11 +59,18 @@ LLVM_ABI void printBumpPtrAllocatorStats(unsigned NumSlabs,
///
/// The GrowthDelay specifies after how many allocated slabs the allocator
/// increases the size of the slabs.
+///
+/// MinAlign keeps the bump pointer aligned between allocations: each size is
+/// rounded up to a multiple of MinAlign so the fast path can skip realigning
+/// CurPtr when the requested alignment is no greater than MinAlign. Costs a
+/// little padding per allocation. Must be 1 for allocators walked at a fixed
+/// stride (see SpecificBumpPtrAllocator).
template <typename AllocatorT = MallocAllocator, size_t SlabSize = 4096,
- size_t SizeThreshold = SlabSize, size_t GrowthDelay = 128>
+ size_t SizeThreshold = SlabSize, size_t GrowthDelay = 128,
+ size_t MinAlign = 8>
class BumpPtrAllocatorImpl
- : public AllocatorBase<BumpPtrAllocatorImpl<AllocatorT, SlabSize,
- SizeThreshold, GrowthDelay>>,
+ : public AllocatorBase<BumpPtrAllocatorImpl<
+ AllocatorT, SlabSize, SizeThreshold, GrowthDelay, MinAlign>>,
private detail::AllocatorHolder<AllocatorT> {
using AllocTy = detail::AllocatorHolder<AllocatorT>;
@@ -75,6 +82,10 @@ class BumpPtrAllocatorImpl
static_assert(GrowthDelay > 0,
"GrowthDelay must be at least 1 which already increases the"
"slab size after each allocated slab.");
+ static_assert(MinAlign > 0 && (MinAlign & (MinAlign - 1)) == 0,
+ "MinAlign must be a power of two.");
+ static_assert(MinAlign <= alignof(std::max_align_t),
+ "MinAlign must not exceed the alignment of fresh slabs.");
BumpPtrAllocatorImpl() = default;
@@ -150,30 +161,31 @@ class BumpPtrAllocatorImpl
// Keep track of how many bytes we've allocated.
BytesAllocated += Size;
- uintptr_t AlignedPtr = alignAddr(CurPtr, Alignment);
-
size_t SizeToAllocate = Size;
#if LLVM_ADDRESS_SANITIZER_BUILD
// Add trailing bytes as a "red zone" under ASan.
SizeToAllocate += RedZoneSize;
#endif
+ SizeToAllocate = alignToPowerOf2(SizeToAllocate, MinAlign);
- uintptr_t AllocEndPtr = AlignedPtr + SizeToAllocate;
- assert(AllocEndPtr >= uintptr_t(CurPtr) &&
+ // CurPtr is already MinAlign-aligned, so only a stricter request realigns.
+ char *Ptr = CurPtr;
+ if (Alignment.value() > MinAlign)
+ Ptr = reinterpret_cast<char *>(alignAddr(Ptr, Alignment));
+ assert(uintptr_t(Ptr) + SizeToAllocate >= uintptr_t(Ptr) &&
"Alignment + Size must not overflow");
- // Check if we have enough space.
- if (LLVM_LIKELY(AllocEndPtr <= uintptr_t(End)
- // We can't return nullptr even for a zero-sized allocation!
- && CurPtr != nullptr)) {
- CurPtr = reinterpret_cast<char *>(AllocEndPtr);
+ // Fast path check. The condition also fails for an empty allocator (End ==
+ // nullptr) to avoid a separate null check.
+ if (LLVM_LIKELY(uintptr_t(Ptr) + SizeToAllocate - 1 < uintptr_t(End))) {
+ CurPtr = Ptr + SizeToAllocate;
// Update the allocation point of this memory block in MemorySanitizer.
// Without this, MemorySanitizer messages for values originated from here
// will point to the allocation of the entire slab.
- __msan_allocated_memory(reinterpret_cast<char *>(AlignedPtr), Size);
+ __msan_allocated_memory(Ptr, Size);
// Similarly, tell ASan about this space.
- __asan_unpoison_memory_region(reinterpret_cast<char *>(AlignedPtr), Size);
- return reinterpret_cast<char *>(AlignedPtr);
+ __asan_unpoison_memory_region(Ptr, Size);
+ return Ptr;
}
return AllocateSlow(Size, SizeToAllocate, Alignment);
@@ -388,7 +400,11 @@ using BumpPtrAllocator = BumpPtrAllocatorImpl<>;
/// This allows calling the destructor in DestroyAll() and when the allocator is
/// destroyed.
template <typename T> class SpecificBumpPtrAllocator {
- BumpPtrAllocator Allocator;
+ // DestroyAll() walks objects at a fixed sizeof(T) stride, so it needs tight
+ // packing: MinAlign=1 disables the default size rounding.
+ using BumpPtrAllocatorTy =
+ BumpPtrAllocatorImpl<MallocAllocator, 4096, 4096, 128, /*MinAlign=*/1>;
+ BumpPtrAllocatorTy Allocator;
public:
SpecificBumpPtrAllocator() {
@@ -417,7 +433,7 @@ template <typename T> class SpecificBumpPtrAllocator {
for (auto I = Allocator.Slabs.begin(), E = Allocator.Slabs.end(); I != E;
++I) {
- size_t AllocatedSlabSize = BumpPtrAllocator::computeSlabSize(
+ size_t AllocatedSlabSize = BumpPtrAllocatorTy::computeSlabSize(
std::distance(Allocator.Slabs.begin(), I));
char *Begin = (char *)alignAddr(*I, Align::Of<T>());
char *End = *I == Allocator.Slabs.back() ? Allocator.CurPtr
@@ -450,20 +466,19 @@ template <typename T> class SpecificBumpPtrAllocator {
} // end namespace llvm
template <typename AllocatorT, size_t SlabSize, size_t SizeThreshold,
- size_t GrowthDelay>
+ size_t GrowthDelay, size_t MinAlign>
void *
operator new(size_t Size,
llvm::BumpPtrAllocatorImpl<AllocatorT, SlabSize, SizeThreshold,
- GrowthDelay> &Allocator) {
- return Allocator.Allocate(Size, std::min((size_t)llvm::NextPowerOf2(Size),
- alignof(std::max_align_t)));
+ GrowthDelay, MinAlign> &Allocator) {
+ return Allocator.Allocate(
+ Size, std::min(llvm::bit_ceil(Size), alignof(std::max_align_t)));
}
template <typename AllocatorT, size_t SlabSize, size_t SizeThreshold,
- size_t GrowthDelay>
-void operator delete(void *,
- llvm::BumpPtrAllocatorImpl<AllocatorT, SlabSize,
- SizeThreshold, GrowthDelay> &) {
-}
+ size_t GrowthDelay, size_t MinAlign>
+void operator delete(
+ void *, llvm::BumpPtrAllocatorImpl<AllocatorT, SlabSize, SizeThreshold,
+ GrowthDelay, MinAlign> &) {}
#endif // LLVM_SUPPORT_ALLOCATOR_H
>From f4e867f260e50704b6dc3e074c447ea36d51bfb1 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sun, 14 Jun 2026 13:32:17 -0700
Subject: [PATCH 2/3] Optimize SpecificBumpPtrAllocator
---
llvm/include/llvm/Support/Allocator.h | 34 +++++++++++++-----------
llvm/unittests/Support/AllocatorTest.cpp | 19 +++++++++++++
2 files changed, 37 insertions(+), 16 deletions(-)
diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h
index eae99d7f8e773..f5e8aae5c8d84 100644
--- a/llvm/include/llvm/Support/Allocator.h
+++ b/llvm/include/llvm/Support/Allocator.h
@@ -62,9 +62,7 @@ LLVM_ABI void printBumpPtrAllocatorStats(unsigned NumSlabs,
///
/// MinAlign keeps the bump pointer aligned between allocations: each size is
/// rounded up to a multiple of MinAlign so the fast path can skip realigning
-/// CurPtr when the requested alignment is no greater than MinAlign. Costs a
-/// little padding per allocation. Must be 1 for allocators walked at a fixed
-/// stride (see SpecificBumpPtrAllocator).
+/// CurPtr when the requested alignment is no greater than MinAlign.
template <typename AllocatorT = MallocAllocator, size_t SlabSize = 4096,
size_t SizeThreshold = SlabSize, size_t GrowthDelay = 128,
size_t MinAlign = 8>
@@ -83,9 +81,9 @@ class BumpPtrAllocatorImpl
"GrowthDelay must be at least 1 which already increases the"
"slab size after each allocated slab.");
static_assert(MinAlign > 0 && (MinAlign & (MinAlign - 1)) == 0,
- "MinAlign must be a power of two.");
+ "MinAlign must be a power of two");
static_assert(MinAlign <= alignof(std::max_align_t),
- "MinAlign must not exceed the alignment of fresh slabs.");
+ "MinAlign must not exceed the alignment of fresh slabs");
BumpPtrAllocatorImpl() = default;
@@ -172,13 +170,15 @@ class BumpPtrAllocatorImpl
char *Ptr = CurPtr;
if (Alignment.value() > MinAlign)
Ptr = reinterpret_cast<char *>(alignAddr(Ptr, Alignment));
- assert(uintptr_t(Ptr) + SizeToAllocate >= uintptr_t(Ptr) &&
- "Alignment + Size must not overflow");
-
- // Fast path check. The condition also fails for an empty allocator (End ==
- // nullptr) to avoid a separate null check.
- if (LLVM_LIKELY(uintptr_t(Ptr) + SizeToAllocate - 1 < uintptr_t(End))) {
- CurPtr = Ptr + SizeToAllocate;
+ char *NewCurPtr = Ptr + SizeToAllocate;
+ assert(NewCurPtr >= Ptr && "Alignment + Size must not overflow");
+
+ // Check if we have enough space.
+ if (LLVM_LIKELY(
+ uintptr_t(NewCurPtr) <= uintptr_t(End) &&
+ // We can't return nullptr even for a zero-sized allocation!
+ CurPtr != nullptr)) {
+ CurPtr = NewCurPtr;
// Update the allocation point of this memory block in MemorySanitizer.
// Without this, MemorySanitizer messages for values originated from here
// will point to the allocation of the entire slab.
@@ -400,10 +400,12 @@ using BumpPtrAllocator = BumpPtrAllocatorImpl<>;
/// This allows calling the destructor in DestroyAll() and when the allocator is
/// destroyed.
template <typename T> class SpecificBumpPtrAllocator {
- // DestroyAll() walks objects at a fixed sizeof(T) stride, so it needs tight
- // packing: MinAlign=1 disables the default size rounding.
+ // MinAlign=alignof(T) keeps DestroyAll()'s fixed sizeof(T) stride tight and,
+ // for the common alignof(T)==8, reuses the default BumpPtrAllocator
+ // instantiation.
using BumpPtrAllocatorTy =
- BumpPtrAllocatorImpl<MallocAllocator, 4096, 4096, 128, /*MinAlign=*/1>;
+ BumpPtrAllocatorImpl<MallocAllocator, 4096, 4096, 128,
+ std::min(alignof(T), alignof(std::max_align_t))>;
BumpPtrAllocatorTy Allocator;
public:
@@ -453,7 +455,7 @@ template <typename T> class SpecificBumpPtrAllocator {
}
/// Allocate space for an array of objects without constructing them.
- T *Allocate(size_t num = 1) { return Allocator.Allocate<T>(num); }
+ T *Allocate(size_t num = 1) { return Allocator.template Allocate<T>(num); }
/// \return An index uniquely and reproducibly identifying
/// an input pointer \p Ptr in the given allocator.
diff --git a/llvm/unittests/Support/AllocatorTest.cpp b/llvm/unittests/Support/AllocatorTest.cpp
index 2337f34143bad..d6f80e0948dc4 100644
--- a/llvm/unittests/Support/AllocatorTest.cpp
+++ b/llvm/unittests/Support/AllocatorTest.cpp
@@ -279,4 +279,23 @@ TEST(AllocatorTest, TestBigAlignment) {
EXPECT_GT(MockSlabAllocator::GetLastSlabSize(), 4096u);
}
+// Over-aligned element type: Allocate() honors alignof(T) > MinAlign and
+// DestroyAll() runs every destructor.
+TEST(AllocatorTest, TestOverAlignedSpecific) {
+ unsigned NumDtorCalls = 0;
+ struct alignas(32) S {
+ unsigned *Calls;
+ ~S() { ++*Calls; }
+ };
+ {
+ SpecificBumpPtrAllocator<S> Alloc;
+ for (int I = 0; I != 4; ++I) {
+ S *P = Alloc.Allocate();
+ EXPECT_EQ(0u, reinterpret_cast<uintptr_t>(P) & 31u);
+ P->Calls = &NumDtorCalls;
+ }
+ }
+ EXPECT_EQ(4u, NumDtorCalls);
+}
+
} // anonymous namespace
>From e3e829a157e1d88bc734edfdf8b18f79cd37ff73 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sun, 14 Jun 2026 15:32:49 -0700
Subject: [PATCH 3/3] Optimize SpecificBumpPtrAllocator
---
llvm/include/llvm/Support/Allocator.h | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h
index f5e8aae5c8d84..e0185bb3a88e6 100644
--- a/llvm/include/llvm/Support/Allocator.h
+++ b/llvm/include/llvm/Support/Allocator.h
@@ -400,12 +400,12 @@ using BumpPtrAllocator = BumpPtrAllocatorImpl<>;
/// This allows calling the destructor in DestroyAll() and when the allocator is
/// destroyed.
template <typename T> class SpecificBumpPtrAllocator {
- // MinAlign=alignof(T) keeps DestroyAll()'s fixed sizeof(T) stride tight and,
- // for the common alignof(T)==8, reuses the default BumpPtrAllocator
- // instantiation.
+ // DestroyAll() walks objects at a fixed sizeof(T) stride, so it needs tight
+ // packing: MinAlign=1 disables the size rounding. (alignof(T) would pack just
+ // as tightly and reuse the default instantiation, but T may be incomplete
+ // here, e.g. SpecificBumpPtrAllocator<MCSectionELF>.)
using BumpPtrAllocatorTy =
- BumpPtrAllocatorImpl<MallocAllocator, 4096, 4096, 128,
- std::min(alignof(T), alignof(std::max_align_t))>;
+ BumpPtrAllocatorImpl<MallocAllocator, 4096, 4096, 128, /*MinAlign=*/1>;
BumpPtrAllocatorTy Allocator;
public:
@@ -455,7 +455,14 @@ template <typename T> class SpecificBumpPtrAllocator {
}
/// Allocate space for an array of objects without constructing them.
- T *Allocate(size_t num = 1) { return Allocator.template Allocate<T>(num); }
+ T *Allocate(size_t num = 1) {
+ // Slabs are max_align_t-aligned and every size is a multiple of alignof(T),
+ // so the bump pointer is already alignof(T)-aligned. Request alignment 1 so
+ // the fast path skips realigning CurPtr; over-aligned T still needs it.
+ if constexpr (alignof(T) <= alignof(std::max_align_t))
+ return static_cast<T *>(Allocator.Allocate(num * sizeof(T), Align()));
+ return Allocator.Allocate<T>(num);
+ }
/// \return An index uniquely and reproducibly identifying
/// an input pointer \p Ptr in the given allocator.
More information about the llvm-commits
mailing list