[llvm] [Allocator] Keep bump pointer at a minimum alignment (PR #203718)
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 13 11:44:47 PDT 2026
https://github.com/MaskRay created https://github.com/llvm/llvm-project/pull/203718
Add a `MinAlign` template parameter (default 8, sizeof(size_t) on 64-bit
platforms) so that the common case `Alignment <= MinAlign` can skip
realigning `CurPtr`.
This is achieved by rounding each allocation's size up to MinAlign, so
the bump pointer stays MinAlign-aligned between allocations.
SpecificBumpPtrAllocator::DestroyAll() walks objects at a fixed sizeof(T)
stride and needs tight packing, so it uses MinAlign=1.
Adjust the fast path check to avoid a separate null check for an emnpty
allocator (`End == nullptr`).
Also use `bit_ceil` instead of `NextPowerOf2` in the placement operator
new alignment heuristic. NextPowerOf2 is strictly greater, so an 8-byte
pointer-sized object asked for alignment 16 and missed the fast path;
bit_ceil(8) == 8 keeps it on the fast path.
>From eec98ef424ed8b6f2fa125e523070d966c6583c6 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Fri, 5 Jun 2026 23:53:22 -0700
Subject: [PATCH] [Allocator] Keep bump pointer at a minimum alignment
Add a `MinAlign` template parameter (default 8, sizeof(size_t) on 64-bit
platforms) so that the common case `Alignment <= MinAlign` can skip
realigning `CurPtr`.
This is achieved by rounding each allocation's size up to MinAlign, so
the bump pointer stays MinAlign-aligned between allocations.
SpecificBumpPtrAllocator::DestroyAll() walks objects at a fixed sizeof(T)
stride and needs tight packing, so it uses MinAlign=1.
Adjust the fast path check to avoid a separate null check for an emnpty
allocator (`End == nullptr`).
Also use `bit_ceil` instead of `NextPowerOf2` in the placement operator
new alignment heuristic. NextPowerOf2 is strictly greater, so an 8-byte
pointer-sized object asked for alignment 16 and missed the fast path;
bit_ceil(8) == 8 keeps it on the fast path.
---
llvm/include/llvm/Support/Allocator.h | 67 ++++++++++++++++-----------
1 file changed, 41 insertions(+), 26 deletions(-)
diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h
index fffcbd9f3c1d8..eae99d7f8e773 100644
--- a/llvm/include/llvm/Support/Allocator.h
+++ b/llvm/include/llvm/Support/Allocator.h
@@ -59,11 +59,18 @@ LLVM_ABI void printBumpPtrAllocatorStats(unsigned NumSlabs,
///
/// The GrowthDelay specifies after how many allocated slabs the allocator
/// increases the size of the slabs.
+///
+/// MinAlign keeps the bump pointer aligned between allocations: each size is
+/// rounded up to a multiple of MinAlign so the fast path can skip realigning
+/// CurPtr when the requested alignment is no greater than MinAlign. Costs a
+/// little padding per allocation. Must be 1 for allocators walked at a fixed
+/// stride (see SpecificBumpPtrAllocator).
template <typename AllocatorT = MallocAllocator, size_t SlabSize = 4096,
- size_t SizeThreshold = SlabSize, size_t GrowthDelay = 128>
+ size_t SizeThreshold = SlabSize, size_t GrowthDelay = 128,
+ size_t MinAlign = 8>
class BumpPtrAllocatorImpl
- : public AllocatorBase<BumpPtrAllocatorImpl<AllocatorT, SlabSize,
- SizeThreshold, GrowthDelay>>,
+ : public AllocatorBase<BumpPtrAllocatorImpl<
+ AllocatorT, SlabSize, SizeThreshold, GrowthDelay, MinAlign>>,
private detail::AllocatorHolder<AllocatorT> {
using AllocTy = detail::AllocatorHolder<AllocatorT>;
@@ -75,6 +82,10 @@ class BumpPtrAllocatorImpl
static_assert(GrowthDelay > 0,
"GrowthDelay must be at least 1 which already increases the"
"slab size after each allocated slab.");
+ static_assert(MinAlign > 0 && (MinAlign & (MinAlign - 1)) == 0,
+ "MinAlign must be a power of two.");
+ static_assert(MinAlign <= alignof(std::max_align_t),
+ "MinAlign must not exceed the alignment of fresh slabs.");
BumpPtrAllocatorImpl() = default;
@@ -150,30 +161,31 @@ class BumpPtrAllocatorImpl
// Keep track of how many bytes we've allocated.
BytesAllocated += Size;
- uintptr_t AlignedPtr = alignAddr(CurPtr, Alignment);
-
size_t SizeToAllocate = Size;
#if LLVM_ADDRESS_SANITIZER_BUILD
// Add trailing bytes as a "red zone" under ASan.
SizeToAllocate += RedZoneSize;
#endif
+ SizeToAllocate = alignToPowerOf2(SizeToAllocate, MinAlign);
- uintptr_t AllocEndPtr = AlignedPtr + SizeToAllocate;
- assert(AllocEndPtr >= uintptr_t(CurPtr) &&
+ // CurPtr is already MinAlign-aligned, so only a stricter request realigns.
+ char *Ptr = CurPtr;
+ if (Alignment.value() > MinAlign)
+ Ptr = reinterpret_cast<char *>(alignAddr(Ptr, Alignment));
+ assert(uintptr_t(Ptr) + SizeToAllocate >= uintptr_t(Ptr) &&
"Alignment + Size must not overflow");
- // Check if we have enough space.
- if (LLVM_LIKELY(AllocEndPtr <= uintptr_t(End)
- // We can't return nullptr even for a zero-sized allocation!
- && CurPtr != nullptr)) {
- CurPtr = reinterpret_cast<char *>(AllocEndPtr);
+ // Fast path check. The condition also fails for an empty allocator (End ==
+ // nullptr) to avoid a separate null check.
+ if (LLVM_LIKELY(uintptr_t(Ptr) + SizeToAllocate - 1 < uintptr_t(End))) {
+ CurPtr = Ptr + SizeToAllocate;
// Update the allocation point of this memory block in MemorySanitizer.
// Without this, MemorySanitizer messages for values originated from here
// will point to the allocation of the entire slab.
- __msan_allocated_memory(reinterpret_cast<char *>(AlignedPtr), Size);
+ __msan_allocated_memory(Ptr, Size);
// Similarly, tell ASan about this space.
- __asan_unpoison_memory_region(reinterpret_cast<char *>(AlignedPtr), Size);
- return reinterpret_cast<char *>(AlignedPtr);
+ __asan_unpoison_memory_region(Ptr, Size);
+ return Ptr;
}
return AllocateSlow(Size, SizeToAllocate, Alignment);
@@ -388,7 +400,11 @@ using BumpPtrAllocator = BumpPtrAllocatorImpl<>;
/// This allows calling the destructor in DestroyAll() and when the allocator is
/// destroyed.
template <typename T> class SpecificBumpPtrAllocator {
- BumpPtrAllocator Allocator;
+ // DestroyAll() walks objects at a fixed sizeof(T) stride, so it needs tight
+ // packing: MinAlign=1 disables the default size rounding.
+ using BumpPtrAllocatorTy =
+ BumpPtrAllocatorImpl<MallocAllocator, 4096, 4096, 128, /*MinAlign=*/1>;
+ BumpPtrAllocatorTy Allocator;
public:
SpecificBumpPtrAllocator() {
@@ -417,7 +433,7 @@ template <typename T> class SpecificBumpPtrAllocator {
for (auto I = Allocator.Slabs.begin(), E = Allocator.Slabs.end(); I != E;
++I) {
- size_t AllocatedSlabSize = BumpPtrAllocator::computeSlabSize(
+ size_t AllocatedSlabSize = BumpPtrAllocatorTy::computeSlabSize(
std::distance(Allocator.Slabs.begin(), I));
char *Begin = (char *)alignAddr(*I, Align::Of<T>());
char *End = *I == Allocator.Slabs.back() ? Allocator.CurPtr
@@ -450,20 +466,19 @@ template <typename T> class SpecificBumpPtrAllocator {
} // end namespace llvm
template <typename AllocatorT, size_t SlabSize, size_t SizeThreshold,
- size_t GrowthDelay>
+ size_t GrowthDelay, size_t MinAlign>
void *
operator new(size_t Size,
llvm::BumpPtrAllocatorImpl<AllocatorT, SlabSize, SizeThreshold,
- GrowthDelay> &Allocator) {
- return Allocator.Allocate(Size, std::min((size_t)llvm::NextPowerOf2(Size),
- alignof(std::max_align_t)));
+ GrowthDelay, MinAlign> &Allocator) {
+ return Allocator.Allocate(
+ Size, std::min(llvm::bit_ceil(Size), alignof(std::max_align_t)));
}
template <typename AllocatorT, size_t SlabSize, size_t SizeThreshold,
- size_t GrowthDelay>
-void operator delete(void *,
- llvm::BumpPtrAllocatorImpl<AllocatorT, SlabSize,
- SizeThreshold, GrowthDelay> &) {
-}
+ size_t GrowthDelay, size_t MinAlign>
+void operator delete(
+ void *, llvm::BumpPtrAllocatorImpl<AllocatorT, SlabSize, SizeThreshold,
+ GrowthDelay, MinAlign> &) {}
#endif // LLVM_SUPPORT_ALLOCATOR_H
More information about the llvm-commits
mailing list