[libc-commits] [libc] [libc] add table based freelist heap implementation (PR #200497)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Fri May 29 13:58:15 PDT 2026
https://github.com/SchrodingerZhu created https://github.com/llvm/llvm-project/pull/200497
- **[libc] add shrink in-place support for reallocations**
- **fix fmt**
- **[libc] add a unit test for shrinking**
- **address code review comments**
- **address format issues**
- **[libc] add table based freelist heap implementation**
>From 6c22212f9b88b6a0991def2bcde4b76fd95273f7 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Thu, 28 May 2026 13:55:52 -0700
Subject: [PATCH 1/6] [libc] add shrink in-place support for reallocations
---
libc/src/__support/block.h | 16 +++---
libc/src/__support/freelist_heap.h | 52 +++++++++++++++++--
libc/src/__support/freestore.h | 2 +
.../test/src/__support/freelist_heap_test.cpp | 3 +-
4 files changed, 62 insertions(+), 11 deletions(-)
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index d45af1079a5bc..399a4db21fa00 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -191,7 +191,8 @@ class Block {
/// field of the next block counts as part of the inner size of the block.
/// `usable_space_alignment` must be a multiple of MIN_ALIGN.
optional<Block *> split(size_t new_inner_size,
- size_t usable_space_alignment = MIN_ALIGN);
+ size_t usable_space_alignment = MIN_ALIGN,
+ size_t min_split_size = sizeof(Block));
/// Merges this block with the one that comes after it.
bool merge_next();
@@ -307,7 +308,8 @@ class Block {
// Divide a block into up to 3 blocks according to `BlockInfo`. Behavior is
// undefined if allocation is not possible for the given size and alignment.
- static BlockInfo allocate(Block *block, size_t alignment, size_t size);
+ static BlockInfo allocate(Block *block, size_t alignment, size_t size,
+ size_t min_split_size = sizeof(Block));
// These two functions may wrap around.
LIBC_INLINE static uintptr_t
@@ -393,7 +395,8 @@ optional<Block *> Block::init(ByteSpan region) {
}
LIBC_INLINE
-Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size) {
+Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size,
+ size_t min_split_size) {
LIBC_ASSERT(alignment % MIN_ALIGN == 0 &&
"alignment must be a multiple of MIN_ALIGN");
@@ -421,7 +424,7 @@ Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size) {
}
// Now get a block for the requested size.
- if (optional<Block *> next = info.block->split(size))
+ if (optional<Block *> next = info.block->split(size, MIN_ALIGN, min_split_size))
info.next = *next;
return info;
@@ -429,7 +432,8 @@ Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size) {
LIBC_INLINE
optional<Block *> Block::split(size_t new_inner_size,
- size_t usable_space_alignment) {
+ size_t usable_space_alignment,
+ size_t min_split_size) {
LIBC_ASSERT(usable_space_alignment % MIN_ALIGN == 0 &&
"alignment must be a multiple of MIN_ALIGN");
if (used())
@@ -449,7 +453,7 @@ optional<Block *> Block::split(size_t new_inner_size,
"new size must be aligned to MIN_ALIGN");
if (outer_size() < new_outer_size ||
- outer_size() - new_outer_size < sizeof(Block))
+ outer_size() - new_outer_size < min_split_size)
return {};
ByteSpan new_region = region().subspan(new_outer_size);
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 66f3739efe214..02d35b1f8ad49 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -53,6 +53,8 @@ class FreeListHeap {
void *allocate_impl(size_t alignment, size_t size);
+ bool shrink_in_place(Block *block, size_t size);
+
span<cpp::byte> block_to_span(Block *block) {
return span<cpp::byte>(block->usable_space(), block->inner_size());
}
@@ -97,7 +99,8 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (!block)
return nullptr;
- auto block_info = Block::allocate(block, alignment, size);
+ auto block_info =
+ Block::allocate(block, alignment, size, FreeStore::MIN_OUTER_SIZE);
if (block_info.next)
free_store.insert(block_info.next);
if (block_info.prev)
@@ -158,6 +161,47 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
free_store.insert(block);
}
+LIBC_INLINE bool FreeListHeap::shrink_in_place(Block *block, size_t size) {
+ size_t min_outer_size = Block::outer_size(cpp::max(size, sizeof(size_t)));
+ uintptr_t next_block_start = Block::next_possible_block_start(
+ reinterpret_cast<uintptr_t>(block) + min_outer_size, Block::MIN_ALIGN);
+ size_t new_outer_size = next_block_start - reinterpret_cast<uintptr_t>(block);
+ // only split the block if the trailing part can be inserted into freelist
+ if (block->outer_size() >= new_outer_size &&
+ block->outer_size() - new_outer_size >= FreeStore::MIN_OUTER_SIZE) {
+ // We must temporarily mark the block as free to allow splitting.
+ // A block's usable space overlaps with the next block's prev_ field. When
+ // the next block is created via splitting, its header constructor writes to
+ // its prev_ field (initializing it to 0). Since the original block is
+ // currently in use, this write will corrupt the last sizeof(size_t) bytes
+ // of active user data. We back up these bytes here and restore them after
+ // the split is completed.
+ cpp::byte *overlap_ptr =
+ reinterpret_cast<cpp::byte *>(block) + new_outer_size;
+ size_t backup;
+ LIBC_NAMESPACE::inline_memcpy(&backup, overlap_ptr, sizeof(size_t));
+ block->mark_free();
+ optional<Block *> next =
+ block->split(size, Block::MIN_ALIGN, FreeStore::MIN_OUTER_SIZE);
+ block->mark_used();
+
+ LIBC_NAMESPACE::inline_memcpy(overlap_ptr, &backup, sizeof(size_t));
+
+ // register the new block on successful split
+ if (next.has_value()) {
+ Block *next_block = *next;
+ Block *right = next_block->next();
+ if (right != nullptr && !right->used()) {
+ free_store.remove(right);
+ next_block->merge_next();
+ }
+ free_store.insert(next_block);
+ }
+ return true;
+ }
+ return false;
+}
+
// Follows constract of the C standard realloc() function
// If ptr is free'd, will return nullptr.
LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
@@ -180,10 +224,10 @@ LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
return nullptr;
size_t old_size = block->inner_size();
- // Do nothing and return ptr if the required memory size is smaller than
- // the current size.
- if (old_size >= size)
+ if (old_size >= size) {
+ shrink_in_place(block, size);
return ptr;
+ }
void *new_ptr = allocate(size);
// Don't invalidate ptr if allocate(size) fails to initilize the memory.
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 2dcb4b10b93d5..d611c17af4644 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -16,6 +16,8 @@ namespace LIBC_NAMESPACE_DECL {
/// A best-fit store of variously-sized free blocks. Blocks can be inserted and
/// removed in logarithmic time.
class FreeStore {
+ friend class FreeListHeap;
+
public:
FreeStore() = default;
FreeStore(const FreeStore &other) = delete;
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 9d3a6b612555f..199408f39dac7 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -180,7 +180,8 @@ TEST_FOR_EACH_ALLOCATOR(ReallocSmallerSize, 2048) {
void *ptr1 = allocator.allocate(ALLOC_SIZE);
void *ptr2 = allocator.realloc(ptr1, kNewAllocSize);
- // For smaller sizes, realloc will not shrink the block.
+ // For smaller sizes, realloc will shrink the block in-place, returning the
+ // same pointer.
EXPECT_EQ(ptr1, ptr2);
}
>From c12967584f9f68fed12f3db0476d078b37d6152e Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Thu, 28 May 2026 13:59:41 -0700
Subject: [PATCH 2/6] fix fmt
---
libc/src/__support/block.h | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index 399a4db21fa00..1da4fd804fee2 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -424,7 +424,8 @@ Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size,
}
// Now get a block for the requested size.
- if (optional<Block *> next = info.block->split(size, MIN_ALIGN, min_split_size))
+ if (optional<Block *> next =
+ info.block->split(size, MIN_ALIGN, min_split_size))
info.next = *next;
return info;
>From 49a59a3c6076bfcd23c2fbb73968221e67a3067f Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Thu, 28 May 2026 15:04:35 -0700
Subject: [PATCH 3/6] [libc] add a unit test for shrinking
---
libc/test/src/__support/CMakeLists.txt | 1 +
.../test/src/__support/freelist_heap_test.cpp | 43 +++++++++++++++++++
2 files changed, 44 insertions(+)
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 6b9c1b4ac8cc7..ea271513beffb 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -69,6 +69,7 @@ if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
libc.src.__support.freelist_heap
libc.src.string.memcmp
libc.src.string.memcpy
+ libc.src.string.memory_utils.inline_memset
)
endif()
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 199408f39dac7..f25a62b056c76 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -11,6 +11,7 @@
#include "src/__support/macros/config.h"
#include "src/string/memcmp.h"
#include "src/string/memcpy.h"
+#include "src/string/memory_utils/inline_memset.h"
#include "test/UnitTest/Test.h"
asm(R"(
@@ -185,6 +186,48 @@ TEST_FOR_EACH_ALLOCATOR(ReallocSmallerSize, 2048) {
EXPECT_EQ(ptr1, ptr2);
}
+TEST_FOR_EACH_ALLOCATOR(ReallocShrinkAndAllocateTrailing, 512) {
+ constexpr size_t ALLOC_SIZE = 300;
+ constexpr size_t NEW_ALLOC_SIZE = 64;
+ constexpr size_t TRAILING_ALLOC_SIZE = 256;
+
+ void *ptr1 = allocator.allocate(ALLOC_SIZE);
+ ASSERT_NE(ptr1, static_cast<void *>(nullptr));
+
+ // Fill the block with a pattern to verify data preservation.
+ LIBC_NAMESPACE::inline_memset(ptr1, 0x5A, ALLOC_SIZE);
+
+ // Shrink the block in-place.
+ void *ptr2 = allocator.realloc(ptr1, NEW_ALLOC_SIZE);
+ ASSERT_NE(ptr2, static_cast<void *>(nullptr));
+ EXPECT_EQ(ptr1, ptr2);
+
+ // Verify that the data in the shrunk block is preserved.
+ unsigned char *uptr2 = static_cast<unsigned char *>(ptr2);
+ for (size_t i = 0; i < NEW_ALLOC_SIZE; ++i)
+ EXPECT_EQ(uptr2[i], static_cast<unsigned char>(0x5A));
+
+ // Allocate another trailing block from the newly split/freed space.
+ void *ptr3 = allocator.allocate(TRAILING_ALLOC_SIZE);
+ ASSERT_NE(ptr3, static_cast<void *>(nullptr));
+
+ // Verify that the trailing block does not overlap with the shrunk block.
+ uintptr_t addr2 = reinterpret_cast<uintptr_t>(ptr2);
+ uintptr_t addr3 = reinterpret_cast<uintptr_t>(ptr3);
+ EXPECT_GE(addr3, addr2 + NEW_ALLOC_SIZE);
+
+ // Verify that writing to the trailing block works without corrupting the
+ // shrunk block.
+ LIBC_NAMESPACE::inline_memset(ptr3, 0xAA, TRAILING_ALLOC_SIZE);
+
+ for (size_t i = 0; i < NEW_ALLOC_SIZE; ++i)
+ EXPECT_EQ(uptr2[i], static_cast<unsigned char>(0x5A));
+
+ // Free the allocations.
+ allocator.free(ptr2);
+ allocator.free(ptr3);
+}
+
TEST_FOR_EACH_ALLOCATOR(ReallocTooLarge, 2048) {
constexpr size_t ALLOC_SIZE = 512;
size_t kNewAllocSize = N * 2; // Large enough to fail.
>From 7c27e1160c21420bd8ba3ef82aa2002101fde7a1 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Fri, 29 May 2026 09:08:57 -0700
Subject: [PATCH 4/6] address code review comments
---
libc/src/__support/block.h | 25 ++++++++-----------
libc/src/__support/freelist_heap.h | 6 ++---
libc/test/src/__support/block_test.cpp | 10 ++++++--
.../test/src/__support/freelist_heap_test.cpp | 11 ++++----
4 files changed, 27 insertions(+), 25 deletions(-)
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index 1da4fd804fee2..86befc0995dd8 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -191,8 +191,7 @@ class Block {
/// field of the next block counts as part of the inner size of the block.
/// `usable_space_alignment` must be a multiple of MIN_ALIGN.
optional<Block *> split(size_t new_inner_size,
- size_t usable_space_alignment = MIN_ALIGN,
- size_t min_split_size = sizeof(Block));
+ size_t usable_space_alignment = MIN_ALIGN);
/// Merges this block with the one that comes after it.
bool merge_next();
@@ -308,8 +307,7 @@ class Block {
// Divide a block into up to 3 blocks according to `BlockInfo`. Behavior is
// undefined if allocation is not possible for the given size and alignment.
- static BlockInfo allocate(Block *block, size_t alignment, size_t size,
- size_t min_split_size = sizeof(Block));
+ static BlockInfo allocate(Block *block, size_t alignment, size_t size);
// These two functions may wrap around.
LIBC_INLINE static uintptr_t
@@ -395,8 +393,7 @@ optional<Block *> Block::init(ByteSpan region) {
}
LIBC_INLINE
-Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size,
- size_t min_split_size) {
+Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size) {
LIBC_ASSERT(alignment % MIN_ALIGN == 0 &&
"alignment must be a multiple of MIN_ALIGN");
@@ -425,7 +422,7 @@ Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size,
// Now get a block for the requested size.
if (optional<Block *> next =
- info.block->split(size, MIN_ALIGN, min_split_size))
+ info.block->split(size, MIN_ALIGN))
info.next = *next;
return info;
@@ -433,12 +430,9 @@ Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size,
LIBC_INLINE
optional<Block *> Block::split(size_t new_inner_size,
- size_t usable_space_alignment,
- size_t min_split_size) {
+ size_t usable_space_alignment) {
LIBC_ASSERT(usable_space_alignment % MIN_ALIGN == 0 &&
"alignment must be a multiple of MIN_ALIGN");
- if (used())
- return {};
// Compute the minimum outer size that produces a block of at least
// `new_inner_size`.
@@ -454,16 +448,19 @@ optional<Block *> Block::split(size_t new_inner_size,
"new size must be aligned to MIN_ALIGN");
if (outer_size() < new_outer_size ||
- outer_size() - new_outer_size < min_split_size)
+ outer_size() - new_outer_size < sizeof(Block))
return {};
+ bool was_free = !used();
+
ByteSpan new_region = region().subspan(new_outer_size);
next_ &= ~SIZE_MASK;
next_ |= new_outer_size;
Block *new_block = as_block(new_region);
- mark_free(); // Free status for this block is now stored in new_block.
- new_block->next()->prev_ = new_region.size();
+ new_block->mark_free();
+ if (was_free)
+ mark_free();
LIBC_ASSERT(new_block->is_usable_space_aligned(usable_space_alignment) &&
"usable space must have requested alignment");
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 02d35b1f8ad49..3bd6df2e8042b 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -100,7 +100,7 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
return nullptr;
auto block_info =
- Block::allocate(block, alignment, size, FreeStore::MIN_OUTER_SIZE);
+ Block::allocate(block, alignment, size);
if (block_info.next)
free_store.insert(block_info.next);
if (block_info.prev)
@@ -180,10 +180,8 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(Block *block, size_t size) {
reinterpret_cast<cpp::byte *>(block) + new_outer_size;
size_t backup;
LIBC_NAMESPACE::inline_memcpy(&backup, overlap_ptr, sizeof(size_t));
- block->mark_free();
optional<Block *> next =
- block->split(size, Block::MIN_ALIGN, FreeStore::MIN_OUTER_SIZE);
- block->mark_used();
+ block->split(size, Block::MIN_ALIGN);
LIBC_NAMESPACE::inline_memcpy(overlap_ptr, &backup, sizeof(size_t));
diff --git a/libc/test/src/__support/block_test.cpp b/libc/test/src/__support/block_test.cpp
index 3029cde834a5d..189e3cab1d784 100644
--- a/libc/test/src/__support/block_test.cpp
+++ b/libc/test/src/__support/block_test.cpp
@@ -248,7 +248,7 @@ TEST(LlvmLibcBlockTest, CanMarkBlockUsed) {
EXPECT_FALSE(block->used());
}
-TEST(LlvmLibcBlockTest, CannotSplitUsedBlock) {
+TEST(LlvmLibcBlockTest, CanSplitUsedBlock) {
constexpr size_t kN = 1024;
constexpr size_t kSplitN = 512;
@@ -259,7 +259,13 @@ TEST(LlvmLibcBlockTest, CannotSplitUsedBlock) {
block->mark_used();
result = block->split(kSplitN);
- ASSERT_FALSE(result.has_value());
+ ASSERT_TRUE(result.has_value());
+ Block *new_block = *result;
+
+ EXPECT_TRUE(block->used());
+ EXPECT_FALSE(new_block->used());
+ EXPECT_EQ(block->next(), new_block);
+ EXPECT_EQ(new_block->prev_free(), static_cast<Block *>(nullptr));
}
TEST(LlvmLibcBlockTest, CanMergeWithNextBlock) {
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index f25a62b056c76..3002834a51619 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -186,10 +186,12 @@ TEST_FOR_EACH_ALLOCATOR(ReallocSmallerSize, 2048) {
EXPECT_EQ(ptr1, ptr2);
}
-TEST_FOR_EACH_ALLOCATOR(ReallocShrinkAndAllocateTrailing, 512) {
- constexpr size_t ALLOC_SIZE = 300;
- constexpr size_t NEW_ALLOC_SIZE = 64;
- constexpr size_t TRAILING_ALLOC_SIZE = 256;
+constexpr size_t SMALL_HEAP_SIZE = 512;
+
+TEST_FOR_EACH_ALLOCATOR(ReallocShrinkAndAllocateTrailing, SMALL_HEAP_SIZE) {
+ constexpr size_t ALLOC_SIZE = (SMALL_HEAP_SIZE * 3) / 4;
+ constexpr size_t NEW_ALLOC_SIZE = SMALL_HEAP_SIZE / 4;
+ constexpr size_t TRAILING_ALLOC_SIZE = SMALL_HEAP_SIZE / 2;
void *ptr1 = allocator.allocate(ALLOC_SIZE);
ASSERT_NE(ptr1, static_cast<void *>(nullptr));
@@ -199,7 +201,6 @@ TEST_FOR_EACH_ALLOCATOR(ReallocShrinkAndAllocateTrailing, 512) {
// Shrink the block in-place.
void *ptr2 = allocator.realloc(ptr1, NEW_ALLOC_SIZE);
- ASSERT_NE(ptr2, static_cast<void *>(nullptr));
EXPECT_EQ(ptr1, ptr2);
// Verify that the data in the shrunk block is preserved.
>From 284cf59591f7b2a35070da5ff9c7965f4b61000e Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Fri, 29 May 2026 09:12:26 -0700
Subject: [PATCH 5/6] address format issues
---
libc/src/__support/block.h | 3 +--
libc/src/__support/freelist_heap.h | 6 ++----
2 files changed, 3 insertions(+), 6 deletions(-)
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index 86befc0995dd8..9be1869652607 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -421,8 +421,7 @@ Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size) {
}
// Now get a block for the requested size.
- if (optional<Block *> next =
- info.block->split(size, MIN_ALIGN))
+ if (optional<Block *> next = info.block->split(size, MIN_ALIGN))
info.next = *next;
return info;
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 3bd6df2e8042b..631eca2d88ac0 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -99,8 +99,7 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (!block)
return nullptr;
- auto block_info =
- Block::allocate(block, alignment, size);
+ auto block_info = Block::allocate(block, alignment, size);
if (block_info.next)
free_store.insert(block_info.next);
if (block_info.prev)
@@ -180,8 +179,7 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(Block *block, size_t size) {
reinterpret_cast<cpp::byte *>(block) + new_outer_size;
size_t backup;
LIBC_NAMESPACE::inline_memcpy(&backup, overlap_ptr, sizeof(size_t));
- optional<Block *> next =
- block->split(size, Block::MIN_ALIGN);
+ optional<Block *> next = block->split(size, Block::MIN_ALIGN);
LIBC_NAMESPACE::inline_memcpy(overlap_ptr, &backup, sizeof(size_t));
>From 7b8f5d0fe994a5fa9f91bf651ec2ee713defe948 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Fri, 29 May 2026 13:57:41 -0700
Subject: [PATCH 6/6] [libc] add table based freelist heap implementation
---
libc/fuzzing/__support/CMakeLists.txt | 23 +-
...reelist_heap_fuzz.cpp => allocator_fuzz.h} | 67 +++--
.../__support/table_freelist_heap_fuzz.cpp | 22 ++
.../__support/trie_freelist_heap_fuzz.cpp | 19 ++
libc/src/__support/CMakeLists.txt | 9 +
libc/src/__support/big_int.h | 9 +
libc/src/__support/freelist.cpp | 3 -
libc/src/__support/freelist.h | 3 +
libc/src/__support/freelist_heap.cpp | 4 +-
libc/src/__support/freelist_heap.h | 80 ++++--
libc/src/__support/freestore.h | 10 +-
libc/src/__support/freetable.h | 269 ++++++++++++++++++
libc/src/__support/math_extras.h | 9 +
libc/test/src/__support/CMakeLists.txt | 13 +
.../test/src/__support/freelist_heap_test.cpp | 147 ++++++----
libc/test/src/__support/freestore_test.cpp | 14 +-
libc/test/src/__support/freetable_test.cpp | 264 +++++++++++++++++
libc/test/src/__support/math_extras_test.cpp | 13 +
18 files changed, 840 insertions(+), 138 deletions(-)
rename libc/fuzzing/__support/{freelist_heap_fuzz.cpp => allocator_fuzz.h} (78%)
create mode 100644 libc/fuzzing/__support/table_freelist_heap_fuzz.cpp
create mode 100644 libc/fuzzing/__support/trie_freelist_heap_fuzz.cpp
create mode 100644 libc/src/__support/freetable.h
create mode 100644 libc/test/src/__support/freetable_test.cpp
diff --git a/libc/fuzzing/__support/CMakeLists.txt b/libc/fuzzing/__support/CMakeLists.txt
index be72259036458..2f16cc3ab8780 100644
--- a/libc/fuzzing/__support/CMakeLists.txt
+++ b/libc/fuzzing/__support/CMakeLists.txt
@@ -38,17 +38,32 @@ add_libc_fuzzer(
# defined by GPU start.cpp files so for now we exclude this fuzzer on GPU.
if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
add_libc_fuzzer(
- freelist_heap_fuzz
+ trie_freelist_heap_fuzz
SRCS
- freelist_heap_fuzz.cpp
+ trie_freelist_heap_fuzz.cpp
DEPENDS
libc.src.__support.freelist_heap
)
# TODO(#119995): Remove this once sccache on Windows no longer requires
# the use of -DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded.
- get_fq_target_name(freelist_heap_fuzz freelist_heap_fuzz_target_name)
+ get_fq_target_name(trie_freelist_heap_fuzz trie_freelist_heap_fuzz_target_name)
set_target_properties(
- ${freelist_heap_fuzz_target_name}
+ ${trie_freelist_heap_fuzz_target_name}
+ PROPERTIES
+ MSVC_DEBUG_INFORMATION_FORMAT ""
+ )
+
+ add_libc_fuzzer(
+ table_freelist_heap_fuzz
+ SRCS
+ table_freelist_heap_fuzz.cpp
+ DEPENDS
+ libc.src.__support.freelist_heap
+ libc.src.__support.freetable
+ )
+ get_fq_target_name(table_freelist_heap_fuzz table_freelist_heap_fuzz_target_name)
+ set_target_properties(
+ ${table_freelist_heap_fuzz_target_name}
PROPERTIES
MSVC_DEBUG_INFORMATION_FORMAT ""
)
diff --git a/libc/fuzzing/__support/freelist_heap_fuzz.cpp b/libc/fuzzing/__support/allocator_fuzz.h
similarity index 78%
rename from libc/fuzzing/__support/freelist_heap_fuzz.cpp
rename to libc/fuzzing/__support/allocator_fuzz.h
index 3675e6c6b7adc..c7bd46179d2d7 100644
--- a/libc/fuzzing/__support/freelist_heap_fuzz.cpp
+++ b/libc/fuzzing/__support/allocator_fuzz.h
@@ -1,4 +1,4 @@
-//===-- freelist_heap_fuzz.cpp --------------------------------------------===//
+//===-- allocator_fuzz.h ----------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,10 +6,13 @@
//
//===----------------------------------------------------------------------===//
///
-/// Fuzzing test for llvm-libc freelist-based heap implementation.
+/// Common templated fuzzing utility for LLVM-libc heap allocators.
///
//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_FUZZING_ALLOCATOR_FUZZ_H
+#define LLVM_LIBC_FUZZING_ALLOCATOR_FUZZ_H
+
#include "src/__support/CPP/bit.h"
#include "src/__support/CPP/optional.h"
#include "src/__support/freelist_heap.h"
@@ -17,21 +20,16 @@
#include "src/string/memory_utils/inline_memmove.h"
#include "src/string/memory_utils/inline_memset.h"
+namespace LIBC_NAMESPACE_DECL {
+
asm(R"(
.globl _end, __llvm_libc_heap_limit
-
.bss
_end:
.fill 1024
__llvm_libc_heap_limit:
)");
-using LIBC_NAMESPACE::Block;
-using LIBC_NAMESPACE::FreeListHeap;
-using LIBC_NAMESPACE::inline_memset;
-using LIBC_NAMESPACE::cpp::nullopt;
-using LIBC_NAMESPACE::cpp::optional;
-
// Record of an outstanding allocation.
struct Alloc {
void *ptr;
@@ -41,9 +39,10 @@ struct Alloc {
};
// A simple vector that tracks allocations using the heap.
+template <typename HeapType>
class AllocVec {
public:
- AllocVec(FreeListHeap &heap) : heap(&heap), size_(0), capacity(0) {
+ AllocVec(HeapType &heap) : heap(&heap), size_(0), capacity(0) {
allocs = nullptr;
}
@@ -68,13 +67,13 @@ class AllocVec {
Alloc &operator[](size_t idx) { return allocs[idx]; }
void erase_idx(size_t idx) {
- LIBC_NAMESPACE::inline_memmove(&allocs[idx], &allocs[idx + 1],
- sizeof(Alloc) * (size_ - idx - 1));
+ inline_memmove(&allocs[idx], &allocs[idx + 1],
+ sizeof(Alloc) * (size_ - idx - 1));
--size_;
}
private:
- FreeListHeap *heap;
+ HeapType *heap;
Alloc *allocs;
size_t size_;
size_t capacity;
@@ -82,11 +81,11 @@ class AllocVec {
// Choose a T value by casting libfuzzer data or exit.
template <typename T>
-optional<T> choose(const uint8_t *&data, size_t &remainder) {
+LIBC_INLINE cpp::optional<T> choose(const uint8_t *&data, size_t &remainder) {
if (sizeof(T) > remainder)
- return nullopt;
+ return cpp::nullopt;
T out;
- LIBC_NAMESPACE::inline_memcpy(&out, data, sizeof(T));
+ inline_memcpy(&out, data, sizeof(T));
data += sizeof(T);
remainder -= sizeof(T);
return out;
@@ -101,31 +100,32 @@ enum class AllocType : uint8_t {
NUM_ALLOC_TYPES,
};
-template <>
-optional<AllocType> choose<AllocType>(const uint8_t *&data, size_t &remainder) {
+template <typename AllocTypeEnum>
+LIBC_INLINE cpp::optional<AllocTypeEnum> choose_alloc_type(const uint8_t *&data, size_t &remainder) {
auto raw = choose<uint8_t>(data, remainder);
if (!raw)
- return nullopt;
- return static_cast<AllocType>(
- *raw % static_cast<uint8_t>(AllocType::NUM_ALLOC_TYPES));
+ return cpp::nullopt;
+ return static_cast<AllocTypeEnum>(
+ *raw % static_cast<uint8_t>(AllocTypeEnum::NUM_ALLOC_TYPES));
}
constexpr size_t heap_size = 64 * 1024;
-optional<size_t> choose_size(const uint8_t *&data, size_t &remainder) {
+LIBC_INLINE cpp::optional<size_t> choose_size(const uint8_t *&data, size_t &remainder) {
auto raw = choose<size_t>(data, remainder);
if (!raw)
- return nullopt;
+ return cpp::nullopt;
return *raw % heap_size;
}
-optional<size_t> choose_alloc_idx(const AllocVec &allocs, const uint8_t *&data,
+template <typename AllocVecType>
+LIBC_INLINE cpp::optional<size_t> choose_alloc_idx(const AllocVecType &allocs, const uint8_t *&data,
size_t &remainder) {
if (allocs.empty())
- return nullopt;
+ return cpp::nullopt;
auto raw = choose<size_t>(data, remainder);
if (!raw)
- return nullopt;
+ return cpp::nullopt;
return *raw % allocs.size();
}
@@ -135,15 +135,16 @@ optional<size_t> choose_alloc_idx(const AllocVec &allocs, const uint8_t *&data,
return 0; \
TYPE NAME = *maybe_##NAME
-extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) {
- LIBC_NAMESPACE::FreeListHeapBuffer<heap_size> heap;
- AllocVec allocs(heap);
+template <typename HeapBufferType>
+LIBC_INLINE int fuzz_one_input(const uint8_t *data, size_t remainder) {
+ HeapBufferType heap;
+ AllocVec<HeapBufferType> allocs(heap);
uint8_t canary = 0;
while (true) {
ASSIGN_OR_RETURN(auto, should_alloc, choose<bool>(data, remainder));
if (should_alloc) {
- ASSIGN_OR_RETURN(auto, alloc_type, choose<AllocType>(data, remainder));
+ ASSIGN_OR_RETURN(auto, alloc_type, choose_alloc_type<AllocType>(data, remainder));
ASSIGN_OR_RETURN(size_t, alloc_size, choose_size(data, remainder));
// Perform allocation.
@@ -155,7 +156,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) {
break;
case AllocType::ALIGNED_ALLOC: {
ASSIGN_OR_RETURN(size_t, alignment, choose_size(data, remainder));
- alignment = LIBC_NAMESPACE::cpp::bit_ceil(alignment);
+ alignment = cpp::bit_ceil(alignment);
ptr = heap.aligned_allocate(alignment, alloc_size);
break;
}
@@ -235,3 +236,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) {
}
return 0;
}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_FUZZING_ALLOCATOR_FUZZ_H
diff --git a/libc/fuzzing/__support/table_freelist_heap_fuzz.cpp b/libc/fuzzing/__support/table_freelist_heap_fuzz.cpp
new file mode 100644
index 0000000000000..ecb32583d0a96
--- /dev/null
+++ b/libc/fuzzing/__support/table_freelist_heap_fuzz.cpp
@@ -0,0 +1,22 @@
+//===-- freetable_heap_fuzz.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// Fuzzing test for llvm-libc FreeTable-based heap implementation.
+///
+//===----------------------------------------------------------------------===//
+
+#include "allocator_fuzz.h"
+#include "src/__support/freetable.h"
+
+using FuzzTable = LIBC_NAMESPACE::FreeTable<32, 3, 2, 3>;
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) {
+ return LIBC_NAMESPACE::fuzz_one_input<
+ LIBC_NAMESPACE::FreeListHeapBuffer<LIBC_NAMESPACE::heap_size, FuzzTable>>(
+ data, remainder);
+}
diff --git a/libc/fuzzing/__support/trie_freelist_heap_fuzz.cpp b/libc/fuzzing/__support/trie_freelist_heap_fuzz.cpp
new file mode 100644
index 0000000000000..7f0b0ce3194e5
--- /dev/null
+++ b/libc/fuzzing/__support/trie_freelist_heap_fuzz.cpp
@@ -0,0 +1,19 @@
+//===-- freelist_heap_fuzz.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// Fuzzing test for llvm-libc freelist-based heap implementation.
+///
+//===----------------------------------------------------------------------===//
+
+#include "allocator_fuzz.h"
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) {
+ return LIBC_NAMESPACE::fuzz_one_input<
+ LIBC_NAMESPACE::FreeListHeapBuffer<LIBC_NAMESPACE::heap_size>>(data,
+ remainder);
+}
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 9d08b4bcf7303..ff1fd7fc9abe1 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -59,6 +59,15 @@ add_header_library(
.freetrie
)
+add_header_library(
+ freetable
+ HDRS
+ freetable.h
+ DEPENDS
+ libc.src.__support.macros.config
+)
+
+
libc_set_definition(libc_freelist_malloc_size "LIBC_FREELIST_MALLOC_SIZE=${LIBC_CONF_FREELIST_MALLOC_BUFFER_SIZE}")
add_object_library(
freelist_heap
diff --git a/libc/src/__support/big_int.h b/libc/src/__support/big_int.h
index 0e5c038ec356e..8bc0a33bf774a 100644
--- a/libc/src/__support/big_int.h
+++ b/libc/src/__support/big_int.h
@@ -1389,6 +1389,15 @@ first_trailing_one(T value) {
return value == 0 ? 0 : cpp::countr_zero(value) + 1;
}
+// Specialization of floor_ilog2 ('math_extras.h') for BigInt.
+template <typename T>
+[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<is_big_int_v<T>, T>
+floor_ilog2(T value) {
+ if (value == 0)
+ return 0;
+ return static_cast<T>(cpp::numeric_limits<T>::digits - cpp::countl_zero(value) - 1);
+}
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_BIG_INT_H
diff --git a/libc/src/__support/freelist.cpp b/libc/src/__support/freelist.cpp
index bfb90ae1c4db4..31d324c893676 100644
--- a/libc/src/__support/freelist.cpp
+++ b/libc/src/__support/freelist.cpp
@@ -12,9 +12,6 @@ namespace LIBC_NAMESPACE_DECL {
void FreeList::push(Node *node) {
if (begin_) {
- LIBC_ASSERT(Block::from_usable_space(node)->outer_size() ==
- begin_->block()->outer_size() &&
- "freelist entries must have the same size");
// Since the list is circular, insert the node immediately before begin_.
node->prev = begin_->prev;
node->next = begin_;
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index c51f14fe57ae7..e64c21a44b390 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -36,6 +36,9 @@ class FreeList {
/// @returns The inner size of blocks in the list containing this node.
LIBC_INLINE size_t size() const { return block()->inner_size(); }
+ LIBC_INLINE Node *next_node() const { return next; }
+ LIBC_INLINE Node *prev_node() const { return prev; }
+
private:
// Circularly linked pointers to adjacent nodes.
Node *prev;
diff --git a/libc/src/__support/freelist_heap.cpp b/libc/src/__support/freelist_heap.cpp
index 4deb0e0f09e22..e3075f52d188f 100644
--- a/libc/src/__support/freelist_heap.cpp
+++ b/libc/src/__support/freelist_heap.cpp
@@ -13,7 +13,7 @@
namespace LIBC_NAMESPACE_DECL {
-static LIBC_CONSTINIT FreeListHeap freelist_heap_symbols;
-FreeListHeap *freelist_heap = &freelist_heap_symbols;
+static LIBC_CONSTINIT FreeListHeap<FreeStore> freelist_heap_symbols;
+FreeListHeap<FreeStore> *freelist_heap = &freelist_heap_symbols;
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 631eca2d88ac0..b254079c8461d 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -1,4 +1,4 @@
-//===-- Interface for freelist_heap ---------------------------------------===//
+//===-- Implementation header for freelist_heap -----------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,21 +6,19 @@
//
//===----------------------------------------------------------------------===//
-#ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST_HEAP_H
-#define LLVM_LIBC_SRC___SUPPORT_FREELIST_HEAP_H
-
-#include <stddef.h>
-
#include "block.h"
#include "freestore.h"
#include "src/__support/CPP/optional.h"
#include "src/__support/CPP/span.h"
+#include "src/__support/CPP/type_traits.h"
#include "src/__support/libc_assert.h"
#include "src/__support/macros/config.h"
-#include "src/__support/math_extras.h"
#include "src/string/memory_utils/inline_memcpy.h"
#include "src/string/memory_utils/inline_memset.h"
+#ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST_HEAP_H
+#define LLVM_LIBC_SRC___SUPPORT_FREELIST_HEAP_H
+
namespace LIBC_NAMESPACE_DECL {
extern "C" cpp::byte _end;
@@ -31,6 +29,15 @@ using cpp::span;
LIBC_INLINE constexpr bool IsPow2(size_t x) { return x && (x & (x - 1)) == 0; }
+template <typename T>
+LIBC_INLINE void init_free_store(T &, size_t) {}
+
+template <>
+LIBC_INLINE void init_free_store<FreeStore>(FreeStore &store, size_t size) {
+ store.set_range({0, cpp::bit_ceil(size)});
+}
+
+template <typename FreeStoreType = FreeStore>
class FreeListHeap {
public:
constexpr FreeListHeap() : begin(&_end), end(&__llvm_libc_heap_limit) {}
@@ -64,27 +71,46 @@ class FreeListHeap {
cpp::byte *begin;
cpp::byte *end;
bool is_initialized = false;
- FreeStore free_store;
+ FreeStoreType free_store;
};
-template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
+// Deduction guide for FreeListHeap to allow bracket-less instantiation
+// e.g. FreeListHeap allocator(buf); -> FreeListHeap<FreeStore>
+FreeListHeap(span<cpp::byte>) -> FreeListHeap<FreeStore>;
+
+template <size_t BUFF_SIZE, typename FreeStoreType = FreeStore>
+class FreeListHeapBuffer : public FreeListHeap<FreeStoreType> {
public:
- constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
+ constexpr FreeListHeapBuffer() : FreeListHeap<FreeStoreType>{buffer}, buffer{} {}
private:
cpp::byte buffer[BUFF_SIZE];
};
-LIBC_INLINE void FreeListHeap::init() {
+// Specialization for the default FreeStore to allow conversion to FreeListHeap*
+template <size_t BUFF_SIZE>
+class FreeListHeapBuffer<BUFF_SIZE, FreeStore> : public FreeListHeap<> {
+public:
+ constexpr FreeListHeapBuffer() : FreeListHeap<>{buffer}, buffer{} {}
+
+private:
+ cpp::byte buffer[BUFF_SIZE];
+};
+
+extern FreeListHeap<FreeStore> *freelist_heap;
+
+template <typename FreeStoreType>
+LIBC_INLINE void FreeListHeap<FreeStoreType>::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = Block::init(region());
Block *block = *result;
- free_store.set_range({0, cpp::bit_ceil(block->inner_size())});
+ init_free_store<FreeStoreType>(free_store, block->inner_size());
free_store.insert(block);
is_initialized = true;
}
-LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
+template <typename FreeStoreType>
+LIBC_INLINE void *FreeListHeap<FreeStoreType>::allocate_impl(size_t alignment, size_t size) {
if (size == 0)
return nullptr;
@@ -95,7 +121,7 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (!request_size)
return nullptr;
- Block *block = free_store.remove_best_fit(request_size);
+ Block *block = free_store.find_and_remove_fit(request_size);
if (!block)
return nullptr;
@@ -109,11 +135,13 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
return block_info.block->usable_space();
}
-LIBC_INLINE void *FreeListHeap::allocate(size_t size) {
+template <typename FreeStoreType>
+LIBC_INLINE void *FreeListHeap<FreeStoreType>::allocate(size_t size) {
return allocate_impl(Block::MIN_ALIGN, size);
}
-LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment,
+template <typename FreeStoreType>
+LIBC_INLINE void *FreeListHeap<FreeStoreType>::aligned_allocate(size_t alignment,
size_t size) {
// The alignment must be an integral power of two.
if (!IsPow2(alignment))
@@ -129,7 +157,8 @@ LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment,
return allocate_impl(alignment, size);
}
-LIBC_INLINE void FreeListHeap::free(void *ptr) {
+template <typename FreeStoreType>
+LIBC_INLINE void FreeListHeap<FreeStoreType>::free(void *ptr) {
if (ptr == nullptr)
return;
@@ -160,14 +189,15 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
free_store.insert(block);
}
-LIBC_INLINE bool FreeListHeap::shrink_in_place(Block *block, size_t size) {
+template <typename FreeStoreType>
+LIBC_INLINE bool FreeListHeap<FreeStoreType>::shrink_in_place(Block *block, size_t size) {
size_t min_outer_size = Block::outer_size(cpp::max(size, sizeof(size_t)));
uintptr_t next_block_start = Block::next_possible_block_start(
reinterpret_cast<uintptr_t>(block) + min_outer_size, Block::MIN_ALIGN);
size_t new_outer_size = next_block_start - reinterpret_cast<uintptr_t>(block);
// only split the block if the trailing part can be inserted into freelist
if (block->outer_size() >= new_outer_size &&
- block->outer_size() - new_outer_size >= FreeStore::MIN_OUTER_SIZE) {
+ block->outer_size() - new_outer_size >= FreeStoreType::MIN_OUTER_SIZE) {
// We must temporarily mark the block as free to allow splitting.
// A block's usable space overlaps with the next block's prev_ field. When
// the next block is created via splitting, its header constructor writes to
@@ -198,9 +228,10 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(Block *block, size_t size) {
return false;
}
-// Follows constract of the C standard realloc() function
+// Follows contract of the C standard realloc() function
// If ptr is free'd, will return nullptr.
-LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
+template <typename FreeStoreType>
+LIBC_INLINE void *FreeListHeap<FreeStoreType>::realloc(void *ptr, size_t size) {
if (size == 0) {
free(ptr);
return nullptr;
@@ -226,7 +257,7 @@ LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
}
void *new_ptr = allocate(size);
- // Don't invalidate ptr if allocate(size) fails to initilize the memory.
+ // Don't invalidate ptr if allocate(size) fails to initialize the memory.
if (new_ptr == nullptr)
return nullptr;
LIBC_NAMESPACE::inline_memcpy(new_ptr, ptr, old_size);
@@ -235,7 +266,8 @@ LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
return new_ptr;
}
-LIBC_INLINE void *FreeListHeap::calloc(size_t num, size_t size) {
+template <typename FreeStoreType>
+LIBC_INLINE void *FreeListHeap<FreeStoreType>::calloc(size_t num, size_t size) {
size_t bytes;
if (__builtin_mul_overflow(num, size, &bytes))
return nullptr;
@@ -245,8 +277,6 @@ LIBC_INLINE void *FreeListHeap::calloc(size_t num, size_t size) {
return ptr;
}
-extern FreeListHeap *freelist_heap;
-
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREELIST_HEAP_H
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index d611c17af4644..3516836ffd989 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -16,7 +16,7 @@ namespace LIBC_NAMESPACE_DECL {
/// A best-fit store of variously-sized free blocks. Blocks can be inserted and
/// removed in logarithmic time.
class FreeStore {
- friend class FreeListHeap;
+ template <typename> friend class FreeListHeap;
public:
FreeStore() = default;
@@ -39,9 +39,9 @@ class FreeStore {
/// Remove a best-fit free block that can contain the given size when
/// allocated. Returns nullptr if there is no such block.
- Block *remove_best_fit(size_t size);
+ Block *find_and_remove_fit(size_t size);
-private:
+public:
static constexpr size_t MIN_OUTER_SIZE =
align_up(sizeof(Block) + sizeof(FreeList::Node), Block::MIN_ALIGN);
static constexpr size_t MIN_LARGE_OUTER_SIZE =
@@ -49,6 +49,8 @@ class FreeStore {
static constexpr size_t NUM_SMALL_SIZES =
(MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / Block::MIN_ALIGN;
+private:
+
LIBC_INLINE static bool too_small(Block *block) {
return block->outer_size() < MIN_OUTER_SIZE;
}
@@ -84,7 +86,7 @@ LIBC_INLINE void FreeStore::remove(Block *block) {
}
}
-LIBC_INLINE Block *FreeStore::remove_best_fit(size_t size) {
+LIBC_INLINE Block *FreeStore::find_and_remove_fit(size_t size) {
if (FreeList *list = find_best_small_fit(size)) {
Block *block = list->front();
list->pop();
diff --git a/libc/src/__support/freetable.h b/libc/src/__support/freetable.h
new file mode 100644
index 0000000000000..a3db6d4c4df02
--- /dev/null
+++ b/libc/src/__support/freetable.h
@@ -0,0 +1,269 @@
+//===-----------------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the definition of the FreeTable class which represents
+/// a table of free lists.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FREETABLE_H
+#define LLVM_LIBC_SRC___SUPPORT_FREETABLE_H
+
+#include "hdr/types/size_t.h"
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/block.h"
+#include "src/__support/common.h"
+#include "src/__support/freelist.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
+#include "src/__support/math_extras.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+// A two-level segregate table for free lists.
+// The table starts with small lists that grows linearly for small sizes, which
+// covers [0, ... UNIT * EXP_BASE].
+// For larger sizes, the bits are managed in a 2-D table.
+// One can think of each row containing NUM_STEPS lists.
+// Along the row, the size grows by 2 exponentially; along the column,
+// the size increases by STEP_SIZE linearly.
+//
+// Mathematical layout:
+// STEP_SIZE = 1 << STEP_SIZE_BITS
+// NUM_STEPS = 1 << NUM_STEP_BITS
+// EXP_BASE = STEP_SIZE * NUM_STEPS
+// LARGE_SIZE_THRESHOLD = UNIT_SIZE * EXP_BASE
+//
+// Visual representation with example parameters:
+// UNIT_SIZE = 32, STEP_SIZE = 8, NUM_STEPS = 4
+// EXP_BASE = 32, THRESHOLD = 1024 B (1 KiB)
+//
+// 1. Small Sizes (Linear Array):
+// Covers [0, ... 1024 B] growing directly by UNIT_SIZE = 32 B
+// ┌───────┬───────┬───────┬───────┬───────┬───────────┬───────────────┐
+// │ [0 B] │ [32B] │ [64B] │ [96B] │ ... │ [992 B] │ [1024 B (Th)] │
+// └───────┴───────┴───────┴───────┴───────┴───────────┴───────────────┘
+//
+// 2. Large Sizes (2-D Table):
+// Rows = FL (Exponential growth), Columns = SL (Linear steps)
+// One can think of each Row containing NUM_STEPS (4) lists.
+//
+// LINEAR INCREASE ALONG COLUMN (SL) --->
+// ┌───────────────┬───────────────┬───────────────┬───────────────┐
+// │ Col = 0 │ Col = 1 │ Col = 2 │ Col = 3 │
+// │ (Base) │ (+25% FL) │ (+50% FL) │ (+75% FL) │
+// ┌─────────┼───────────────┼───────────────┼───────────────┼───────────────┤
+// E │ Row = 0 │ 1024 B │ 1280 B │ 1536 B │ 1792 B │
+// X │(Base 1K)│ [1024 - 1279] │ [1280 - 1535] │ [1536 - 1791] │ [1792 - 2047] │
+// P ├─────────┼───────────────┼───────────────┼───────────────┼───────────────┤
+// O │ Row = 1 │ 2048 B │ 2560 B │ 3072 B │ 3584 B │
+// N │(Base 2K)│ [2048 - 2559] │ [2560 - 3071] │ [3072 - 3583] │ [3584 - 4095] │
+// E ├─────────┼───────────────┼───────────────┼───────────────┼───────────────┤
+// N │ Row = 2 │ 4096 B │ 5120 B │ 6144 B │ 7168 B │
+// T │(Base 4K)│ [4096 - 5119] │ [5120 - 6143] │ [6144 - 7167] │ [7168 - 8191] │
+// I ├─────────┼───────────────┼───────────────┼───────────────┼───────────────┤
+// A │ Row = 3 │ 8192 B │ 10240 B │ 12288 B │ 14336 B │
+// L │(Base 8K)│ [8192 - 10239]│[10240 - 12287]│[12288 - 14335]│[14336 - 16383]│
+// └─────────┴───────────────┴───────────────┴───────────────┴───────────────┘
+//
+// Note: For the real implementation, we don't actually store the lists in a 2-D
+// structure. Instead, we flatten the entire 2-D layout into a single flat 1-D
+// array of size `TOTAL_BITS` (`free_lists`), and map sizes directly to a
+// continuous 1-D index using `size_to_bit_index`. The allocation state is
+// tracked compactly in the `lookup_table` bitmask array.
+
+// Separate config object to avoid repeated template parameters.
+template <size_t UNIT_SIZE_VAL, size_t STEP_SIZE_BITS_VAL,
+ size_t NUM_STEP_BITS_VAL, size_t NUM_TABLE_ENTRIES_VAL>
+struct FreeTableConfig {
+ // size of basic allocation units, should be power of 2
+ constexpr static size_t UNIT_SIZE = UNIT_SIZE_VAL;
+ // log2 of linear division step size
+ constexpr static size_t STEP_SIZE_BITS = STEP_SIZE_BITS_VAL;
+ // log2 of number of steps per exponential level
+ constexpr static size_t NUM_STEP_BITS = NUM_STEP_BITS_VAL;
+ // total number of entries in the table
+ constexpr static size_t NUM_TABLE_ENTRIES = NUM_TABLE_ENTRIES_VAL;
+};
+
+template <typename CONFIG> struct FreeTableImpl {
+protected:
+ constexpr static size_t STEP_SIZE = size_t(1) << CONFIG::STEP_SIZE_BITS;
+ constexpr static size_t NUM_STEPS = size_t(1) << CONFIG::NUM_STEP_BITS;
+ constexpr static size_t EXP_BASE = STEP_SIZE * NUM_STEPS;
+ constexpr static size_t LARGE_SIZE_THRESHOLD = CONFIG::UNIT_SIZE * EXP_BASE;
+ constexpr static size_t BITS_PER_ENTRY =
+ cpp::numeric_limits<uintptr_t>::digits;
+ constexpr static size_t TOTAL_BITS =
+ CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
+public:
+ constexpr static size_t MIN_OUTER_SIZE =
+ align_up(sizeof(Block) + sizeof(FreeList::Node), Block::MIN_ALIGN);
+
+protected:
+
+ LIBC_INLINE static bool too_small(Block *block) {
+ return block->outer_size() < MIN_OUTER_SIZE;
+ }
+
+ cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
+ cpp::array<FreeList, TOTAL_BITS> free_lists{};
+
+ LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
+ LIBC_INLINE void set_bit(size_t bit_index);
+ LIBC_INLINE void clear_bit(size_t bit_index);
+ LIBC_INLINE bool get_bit(size_t bit_index) const;
+ LIBC_INLINE size_t find_first_bit_set_after(size_t bit_index) const;
+ LIBC_INLINE Block *remove_first_fit_in_list(size_t index, size_t size);
+
+public:
+ LIBC_INLINE void insert(Block *block);
+ LIBC_INLINE void remove(Block *block);
+ LIBC_INLINE Block *find_and_remove_fit(size_t size);
+ LIBC_INLINE FreeTableImpl() = default;
+};
+
+template <typename CONFIG>
+LIBC_INLINE constexpr size_t
+FreeTableImpl<CONFIG>::size_to_bit_index(size_t size) {
+ size_t shifted_size = size / CONFIG::UNIT_SIZE;
+ size_t index = shifted_size;
+ if (shifted_size > EXP_BASE) {
+ size_t exp_index = floor_ilog2(shifted_size / EXP_BASE);
+ size_t base_shifted = EXP_BASE << exp_index;
+ size_t step_shifted = base_shifted >> CONFIG::NUM_STEP_BITS;
+ size_t linear_index = (shifted_size - base_shifted) / step_shifted;
+ index = EXP_BASE + NUM_STEPS * exp_index + linear_index;
+ }
+ return index < TOTAL_BITS ? index : TOTAL_BITS - 1;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void FreeTableImpl<CONFIG>::set_bit(size_t bit_index) {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ lookup_table[entry_index] |= (uintptr_t(1) << bit_offset);
+}
+
+template <typename CONFIG>
+LIBC_INLINE void FreeTableImpl<CONFIG>::clear_bit(size_t bit_index) {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ lookup_table[entry_index] &= ~(uintptr_t(1) << bit_offset);
+}
+
+template <typename CONFIG>
+LIBC_INLINE bool FreeTableImpl<CONFIG>::get_bit(size_t bit_index) const {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ return (lookup_table[entry_index] & (uintptr_t(1) << bit_offset)) != 0;
+}
+
+template <typename CONFIG>
+LIBC_INLINE size_t
+FreeTableImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
+ size_t target_index = bit_index + 1;
+ if (target_index >= TOTAL_BITS)
+ return TOTAL_BITS;
+ size_t start_entry = target_index / BITS_PER_ENTRY;
+ size_t bit_offset = target_index % BITS_PER_ENTRY;
+
+ uintptr_t val = lookup_table[start_entry] & (~uintptr_t(0) << bit_offset);
+ if (val != 0)
+ return start_entry * BITS_PER_ENTRY +
+ static_cast<size_t>(cpp::countr_zero(val));
+
+ for (size_t i = start_entry + 1; i < CONFIG::NUM_TABLE_ENTRIES; ++i) {
+ uintptr_t v = lookup_table[i];
+ if (v != 0)
+ return i * BITS_PER_ENTRY + static_cast<size_t>(cpp::countr_zero(v));
+ }
+ return TOTAL_BITS;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void FreeTableImpl<CONFIG>::insert(Block *block) {
+ if (too_small(block))
+ return;
+ size_t bit_index = size_to_bit_index(block->inner_size());
+ free_lists[bit_index].push(block);
+ set_bit(bit_index);
+}
+
+template <typename CONFIG>
+LIBC_INLINE void FreeTableImpl<CONFIG>::remove(Block *block) {
+ if (too_small(block))
+ return;
+ size_t bit_index = size_to_bit_index(block->inner_size());
+ free_lists[bit_index].remove(
+ reinterpret_cast<FreeList::Node *>(block->usable_space()));
+ if (free_lists[bit_index].empty())
+ clear_bit(bit_index);
+}
+
+template <typename CONFIG>
+LIBC_INLINE Block *
+FreeTableImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
+ // Performs a linear search on the free list to find the first block that fits.
+ // Note that this linear search only ever happens during large allocations when
+ // falling back to searching the exact-fit size class (or the overflow bin).
+ // For standard small allocations, we always find a guaranteed fit in a larger
+ // oversized freelist, allowing a one-shot O(1) pop.
+ FreeList::Node *begin_node = free_lists[index].begin();
+ if (begin_node == nullptr)
+ return nullptr;
+
+ FreeList::Node *cur = begin_node;
+ do {
+ if (cur->size() >= size) {
+ free_lists[index].remove(cur);
+ if (free_lists[index].empty())
+ clear_bit(index);
+ return cur->block();
+ }
+ cur = cur->next_node();
+ } while (cur != begin_node);
+
+ return nullptr;
+}
+
+template <typename CONFIG>
+LIBC_INLINE Block *FreeTableImpl<CONFIG>::find_and_remove_fit(size_t size) {
+ size_t bit_index = size_to_bit_index(size);
+ // If the computed bit index overflows the table structure, fallback to
+ // searching the last freelist (which serves as the remainder/overflow bin).
+ if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1))
+ return remove_first_fit_in_list(TOTAL_BITS - 1, size);
+
+ // Search for the first oversized free list that can guarantee a fit.
+ size_t first_oversized_bit = find_first_bit_set_after(bit_index);
+ // If no larger free list has blocks, fallback to searching the exact free
+ // list.
+ if (LIBC_UNLIKELY(first_oversized_bit >= TOTAL_BITS))
+ return remove_first_fit_in_list(bit_index, size);
+
+ // If a larger free list is found, any block inside it is guaranteed to fit
+ // the requested size. Pop the first block in FIFO order.
+ Block *block = free_lists[first_oversized_bit].front();
+ free_lists[first_oversized_bit].pop();
+ if (free_lists[first_oversized_bit].empty())
+ clear_bit(first_oversized_bit);
+ return block;
+}
+
+template <size_t UNIT_SIZE, size_t STEP_SIZE_BITS, size_t NUM_STEP_BITS,
+ size_t NUM_TABLE_ENTRIES>
+using FreeTable =
+ FreeTableImpl<FreeTableConfig<UNIT_SIZE, STEP_SIZE_BITS, NUM_STEP_BITS,
+ NUM_TABLE_ENTRIES>>;
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FREETABLE_H
diff --git a/libc/src/__support/math_extras.h b/libc/src/__support/math_extras.h
index b8abf8041c8e7..e330b22bab531 100644
--- a/libc/src/__support/math_extras.h
+++ b/libc/src/__support/math_extras.h
@@ -180,6 +180,15 @@ first_trailing_one(T value) {
return value == 0 ? 0 : cpp::countr_zero(value) + 1;
}
+template <typename T>
+[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_unsigned_v<T>, T>
+floor_ilog2(T value) {
+ if (value == 0)
+ return 0;
+ return static_cast<T>(cpp::numeric_limits<T>::digits -
+ cpp::countl_zero(value) - 1);
+}
+
template <typename T>
[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_unsigned_v<T>, int>
count_zeros(T value) {
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index ea271513beffb..a5c40f2b834e3 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -53,6 +53,18 @@ if(NOT LIBC_TARGET_OS_IS_GPU)
libc.src.__support.freestore
libc.src.__support.freetrie
)
+
+ add_libc_test(
+ freetable_test
+ SUITE
+ libc-support-tests
+ SRCS
+ freetable_test.cpp
+ DEPENDS
+ libc.src.__support.freetable
+ libc.src.__support.block
+ libc.src.__support.freelist
+ )
endif()
# TODO: FreeListHeap uses the _end symbol which conflicts with the _end symbol
@@ -67,6 +79,7 @@ if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
DEPENDS
libc.src.__support.CPP.span
libc.src.__support.freelist_heap
+ libc.src.__support.freetable
libc.src.string.memcmp
libc.src.string.memcpy
libc.src.string.memory_utils.inline_memset
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 3002834a51619..6a192d19411a8 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -8,6 +8,7 @@
#include "src/__support/CPP/span.h"
#include "src/__support/freelist_heap.h"
+#include "src/__support/freetable.h"
#include "src/__support/macros/config.h"
#include "src/string/memcmp.h"
#include "src/string/memcpy.h"
@@ -16,7 +17,6 @@
asm(R"(
.globl _end, __llvm_libc_heap_limit
-
.bss
_end:
.fill 1024
@@ -27,38 +27,56 @@ using LIBC_NAMESPACE::Block;
using LIBC_NAMESPACE::freelist_heap;
using LIBC_NAMESPACE::FreeListHeap;
using LIBC_NAMESPACE::FreeListHeapBuffer;
+using LIBC_NAMESPACE::FreeTable;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::span;
-// Similar to `LlvmLibcBlockTest` in block_test.cpp, we'd like to run the same
-// tests independently for different parameters. In this case, we'd like to test
-// functionality for a `FreeListHeap` and the global `freelist_heap` which was
-// constinit'd. Functionally, it should operate the same if the FreeListHeap
-// were initialized locally at runtime or at compile-time.
-//
-// Note that calls to `allocate` for each test case here don't always explicitly
-// `free` them afterwards, so when testing the global allocator, allocations
-// made in tests leak and aren't free'd. This is fine for the purposes of this
-// test file.
+// A test FreeTable configuration (192 bits table)
+using TestTable = FreeTable<32, 3, 2, 3>;
+
+// Macro that generates a test fixture that runs the given test logic
+// against both the FreeStore (Trie) and FreeTable (TLSF) backed heaps!
#define TEST_FOR_EACH_ALLOCATOR(TestCase, BufferSize) \
class LlvmLibcFreeListHeapTest##TestCase \
: public LIBC_NAMESPACE::testing::Test { \
public: \
- FreeListHeapBuffer<BufferSize> fake_global_buffer; \
- void SetUp() override { \
- freelist_heap = \
- new (&fake_global_buffer) FreeListHeapBuffer<BufferSize>; \
- } \
- void RunTest(FreeListHeap &allocator, [[maybe_unused]] size_t N); \
+ FreeListHeapBuffer<BufferSize, LIBC_NAMESPACE::FreeStore> \
+ fake_global_buffer_store; \
+ FreeListHeapBuffer<BufferSize, TestTable> fake_global_buffer_table; \
+ void RunTestStore(FreeListHeap<LIBC_NAMESPACE::FreeStore> &allocator, \
+ size_t N); \
+ void RunTestTable(FreeListHeap<TestTable> &allocator, size_t N); \
+ template <typename allocator_type> \
+ void RunTestImpl(allocator_type &allocator, [[maybe_unused]] size_t N); \
}; \
TEST_F(LlvmLibcFreeListHeapTest##TestCase, TestCase) { \
- byte buf[BufferSize] = {byte(0)}; \
- FreeListHeap allocator(buf); \
- RunTest(allocator, BufferSize); \
- RunTest(*freelist_heap, freelist_heap->region().size()); \
+ { \
+ byte buf[BufferSize] = {byte(0)}; \
+ FreeListHeap<LIBC_NAMESPACE::FreeStore> allocator(buf); \
+ RunTestStore(allocator, BufferSize); \
+ } \
+ { \
+ byte buf[BufferSize] = {byte(0)}; \
+ FreeListHeap<TestTable> allocator(buf); \
+ RunTestTable(allocator, BufferSize); \
+ } \
+ { \
+ freelist_heap = new (&fake_global_buffer_store) \
+ FreeListHeapBuffer<BufferSize, LIBC_NAMESPACE::FreeStore>; \
+ RunTestStore(*freelist_heap, freelist_heap->region().size()); \
+ } \
+ } \
+ void LlvmLibcFreeListHeapTest##TestCase::RunTestStore( \
+ FreeListHeap<LIBC_NAMESPACE::FreeStore> &allocator, size_t N) { \
+ RunTestImpl(allocator, N); \
} \
- void LlvmLibcFreeListHeapTest##TestCase::RunTest(FreeListHeap &allocator, \
- [[maybe_unused]] size_t N)
+ void LlvmLibcFreeListHeapTest##TestCase::RunTestTable( \
+ FreeListHeap<TestTable> &allocator, size_t N) { \
+ RunTestImpl(allocator, N); \
+ } \
+ template <typename allocator_type> \
+ void LlvmLibcFreeListHeapTest##TestCase::RunTestImpl( \
+ allocator_type &allocator, [[maybe_unused]] size_t N)
TEST_FOR_EACH_ALLOCATOR(CanAllocate, 2048) {
constexpr size_t ALLOC_SIZE = 512;
@@ -85,8 +103,6 @@ TEST_FOR_EACH_ALLOCATOR(AllocationsDontOverlap, 2048) {
}
TEST_FOR_EACH_ALLOCATOR(CanFreeAndRealloc, 2048) {
- // There's not really a nice way to test that free works, apart from to try
- // and get that value back again.
constexpr size_t ALLOC_SIZE = 512;
void *ptr1 = allocator.allocate(ALLOC_SIZE);
@@ -100,24 +116,50 @@ TEST_FOR_EACH_ALLOCATOR(ReturnsNullWhenAllocationTooLarge, 2048) {
EXPECT_EQ(allocator.allocate(N), static_cast<void *>(nullptr));
}
-// NOTE: This doesn't use TEST_FOR_EACH_ALLOCATOR because the first `allocate`
-// here will likely actually return a nullptr since the same global allocator
-// is used for other test cases and we don't explicitly free them.
-TEST(LlvmLibcFreeListHeap, ReturnsNullWhenFull) {
- constexpr size_t N = 2048;
- byte buf[N];
+class LlvmLibcFreeListHeapHelper : public LIBC_NAMESPACE::testing::Test {
+public:
+ template <typename allocator_type> void test_returns_null_when_full() {
+ constexpr size_t N = 2048;
+ byte buf[N];
+
+ allocator_type allocator(buf);
+
+ bool went_null = false;
+ for (size_t i = 0; i < N; i++) {
+ if (!allocator.allocate(1)) {
+ went_null = true;
+ break;
+ }
+ }
+ EXPECT_TRUE(went_null);
+ EXPECT_EQ(allocator.allocate(1), static_cast<void *>(nullptr));
+ }
+
+ template <typename allocator_type>
+ void test_aligned_alloc_unaligned_buffer() {
+ byte buf[4096] = {byte(0)};
+
+ // Ensure the underlying buffer is poorly aligned.
+ allocator_type allocator(span<byte>(buf).subspan(1));
- FreeListHeap allocator(buf);
+ constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
+ constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
- bool went_null = false;
- for (size_t i = 0; i < N; i++) {
- if (!allocator.allocate(1)) {
- went_null = true;
- break;
+ for (size_t alignment : ALIGNMENTS) {
+ for (size_t scale : SIZE_SCALES) {
+ size_t size = alignment * scale;
+ void *ptr = allocator.aligned_allocate(alignment, size);
+ EXPECT_NE(ptr, static_cast<void *>(nullptr));
+ EXPECT_EQ(reinterpret_cast<uintptr_t>(ptr) % alignment, size_t(0));
+ allocator.free(ptr);
+ }
}
}
- EXPECT_TRUE(went_null);
- EXPECT_EQ(allocator.allocate(1), static_cast<void *>(nullptr));
+};
+
+TEST_F(LlvmLibcFreeListHeapHelper, ReturnsNullWhenFull) {
+ test_returns_null_when_full<FreeListHeap<LIBC_NAMESPACE::FreeStore>>();
+ test_returns_null_when_full<FreeListHeap<TestTable>>();
}
TEST_FOR_EACH_ALLOCATOR(ReturnedPointersAreAligned, 2048) {
@@ -294,29 +336,10 @@ TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 2048) {
}
}
-// This test is not part of the TEST_FOR_EACH_ALLOCATOR since we want to
-// explicitly ensure that the buffer can still return aligned allocations even
-// if the underlying buffer is unaligned. This is so we can check that we can
-// still get aligned allocations even if the underlying buffer is not aligned to
-// the alignments we request.
-TEST(LlvmLibcFreeListHeap, AlignedAllocUnalignedBuffer) {
- byte buf[4096] = {byte(0)};
-
- // Ensure the underlying buffer is poorly aligned.
- FreeListHeap allocator(span<byte>(buf).subspan(1));
-
- constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
- constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
-
- for (size_t alignment : ALIGNMENTS) {
- for (size_t scale : SIZE_SCALES) {
- size_t size = alignment * scale;
- void *ptr = allocator.aligned_allocate(alignment, size);
- EXPECT_NE(ptr, static_cast<void *>(nullptr));
- EXPECT_EQ(reinterpret_cast<uintptr_t>(ptr) % alignment, size_t(0));
- allocator.free(ptr);
- }
- }
+TEST_F(LlvmLibcFreeListHeapHelper, AlignedAllocUnalignedBuffer) {
+ test_aligned_alloc_unaligned_buffer<
+ FreeListHeap<LIBC_NAMESPACE::FreeStore>>();
+ test_aligned_alloc_unaligned_buffer<FreeListHeap<TestTable>>();
}
TEST_FOR_EACH_ALLOCATOR(InvalidAlignedAllocAlignment, 2048) {
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index 7017d6b9ebe93..12cd0fcc6fc0b 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -37,7 +37,7 @@ TEST(LlvmLibcFreeStore, TooSmall) {
store.insert(too_small);
store.insert(remainder);
- EXPECT_EQ(store.remove_best_fit(too_small->inner_size()), remainder);
+ EXPECT_EQ(store.find_and_remove_fit(too_small->inner_size()), remainder);
store.remove(too_small);
}
@@ -68,20 +68,20 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) {
store.insert(remainder);
// Find exact match for smallest.
- ASSERT_EQ(store.remove_best_fit(smallest->inner_size()), smallest);
+ ASSERT_EQ(store.find_and_remove_fit(smallest->inner_size()), smallest);
store.insert(smallest);
// Find exact match for largest.
- ASSERT_EQ(store.remove_best_fit(largest_small->inner_size()), largest_small);
+ ASSERT_EQ(store.find_and_remove_fit(largest_small->inner_size()), largest_small);
store.insert(largest_small);
// Search small list for best fit.
Block *next_smallest = largest_small == smallest ? remainder : largest_small;
- ASSERT_EQ(store.remove_best_fit(smallest->inner_size() + 1), next_smallest);
+ ASSERT_EQ(store.find_and_remove_fit(smallest->inner_size() + 1), next_smallest);
store.insert(next_smallest);
// Continue search for best fit to large blocks.
- EXPECT_EQ(store.remove_best_fit(largest_small->inner_size() + 1), remainder);
+ EXPECT_EQ(store.find_and_remove_fit(largest_small->inner_size() + 1), remainder);
}
TEST(LlvmLibcFreeStore, Remove) {
@@ -101,9 +101,9 @@ TEST(LlvmLibcFreeStore, Remove) {
store.insert(remainder);
store.remove(remainder);
- ASSERT_EQ(store.remove_best_fit(remainder->inner_size()),
+ ASSERT_EQ(store.find_and_remove_fit(remainder->inner_size()),
static_cast<Block *>(nullptr));
store.remove(small);
- ASSERT_EQ(store.remove_best_fit(small->inner_size()),
+ ASSERT_EQ(store.find_and_remove_fit(small->inner_size()),
static_cast<Block *>(nullptr));
}
diff --git a/libc/test/src/__support/freetable_test.cpp b/libc/test/src/__support/freetable_test.cpp
new file mode 100644
index 0000000000000..f465f21546fea
--- /dev/null
+++ b/libc/test/src/__support/freetable_test.cpp
@@ -0,0 +1,264 @@
+//===-----------------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for a FreeTable.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/limits.h"
+#include "src/__support/block.h"
+#include "src/__support/freetable.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::Block;
+using LIBC_NAMESPACE::FreeTable;
+
+// A test helper struct that inherits from FreeTable to expose its protected
+// internal helper methods to the unit tests.
+template <size_t UNIT_SIZE, size_t STEP_SIZE_BITS, size_t NUM_STEP_BITS,
+ size_t NUM_TABLE_ENTRIES>
+struct TestTable : public FreeTable<UNIT_SIZE, STEP_SIZE_BITS, NUM_STEP_BITS, NUM_TABLE_ENTRIES> {
+ using Base = FreeTable<UNIT_SIZE, STEP_SIZE_BITS, NUM_STEP_BITS, NUM_TABLE_ENTRIES>;
+ using Base::TOTAL_BITS;
+ using Base::size_to_bit_index;
+ using Base::get_bit;
+ using Base::find_first_bit_set_after;
+ using Base::set_bit;
+ using Base::clear_bit;
+ using Base::remove_first_fit_in_list;
+};
+
+TEST(LlvmLibcFreeTableTest, SizeToBitIndex) {
+ constexpr size_t BITS_PER_ENTRY = LIBC_NAMESPACE::cpp::numeric_limits<uintptr_t>::digits;
+ constexpr size_t NUM_TABLE_ENTRIES = 192 / BITS_PER_ENTRY;
+
+ // Parameters matching our visual/mathematical layout example:
+ // UNIT_SIZE = 32, STEP_SIZE = 8, NUM_STEPS = 4
+ using Table = TestTable<32, 3, 2, NUM_TABLE_ENTRIES>;
+
+ // 1. Small sizes (linear region):
+ EXPECT_EQ(Table::size_to_bit_index(0), size_t(0));
+ EXPECT_EQ(Table::size_to_bit_index(31), size_t(0));
+ EXPECT_EQ(Table::size_to_bit_index(32), size_t(1));
+ EXPECT_EQ(Table::size_to_bit_index(63), size_t(1));
+ EXPECT_EQ(Table::size_to_bit_index(64), size_t(2));
+ EXPECT_EQ(Table::size_to_bit_index(992), size_t(31));
+ EXPECT_EQ(Table::size_to_bit_index(1024), size_t(32));
+
+ // 2. Large sizes (2-D exponential region):
+ // Row 0 (Base 1024):
+ // Col 0: [1024, 1279] -> bit index 32
+ EXPECT_EQ(Table::size_to_bit_index(1025), size_t(32));
+ EXPECT_EQ(Table::size_to_bit_index(1279), size_t(32));
+ // Col 1: [1280, 1535] -> bit index 33
+ EXPECT_EQ(Table::size_to_bit_index(1280), size_t(33));
+ EXPECT_EQ(Table::size_to_bit_index(1535), size_t(33));
+ // Col 2: [1536, 1791] -> bit index 34
+ EXPECT_EQ(Table::size_to_bit_index(1536), size_t(34));
+ EXPECT_EQ(Table::size_to_bit_index(1791), size_t(34));
+ // Col 3: [1792, 2047] -> bit index 35
+ EXPECT_EQ(Table::size_to_bit_index(1792), size_t(35));
+ EXPECT_EQ(Table::size_to_bit_index(2047), size_t(35));
+
+ // Row 1 (Base 2048):
+ // Col 0: [2048, 2559] -> bit index 36
+ EXPECT_EQ(Table::size_to_bit_index(2048), size_t(36));
+ EXPECT_EQ(Table::size_to_bit_index(2559), size_t(36));
+ // Col 1: [2560, 3071] -> bit index 37
+ EXPECT_EQ(Table::size_to_bit_index(2560), size_t(37));
+ EXPECT_EQ(Table::size_to_bit_index(3000), size_t(37));
+ EXPECT_EQ(Table::size_to_bit_index(3071), size_t(37));
+
+ // Row 2 (Base 4096):
+ EXPECT_EQ(Table::size_to_bit_index(4096), size_t(40));
+ EXPECT_EQ(Table::size_to_bit_index(5120), size_t(41));
+
+ // 3. Clamping of extremely large sizes to Table::TOTAL_BITS - 1 (191):
+ if constexpr (sizeof(size_t) == 8) {
+ EXPECT_EQ(Table::size_to_bit_index(1ULL << 52), Table::TOTAL_BITS - 1);
+ } else {
+ EXPECT_EQ(Table::size_to_bit_index(4294967295ULL), size_t(120));
+ }
+}
+
+TEST(LlvmLibcFreeTableTest, BitManipulation) {
+ constexpr size_t BITS_PER_ENTRY = LIBC_NAMESPACE::cpp::numeric_limits<uintptr_t>::digits;
+ constexpr size_t NUM_TABLE_ENTRIES = 192 / BITS_PER_ENTRY;
+
+ // 192-bit table
+ using Table = TestTable<32, 3, 2, NUM_TABLE_ENTRIES>;
+ Table table;
+
+ // Initially all bits should be 0
+ for (size_t i = 0; i < Table::TOTAL_BITS; ++i)
+ EXPECT_FALSE(table.get_bit(i));
+
+ // Test set_bit / get_bit
+ table.set_bit(10);
+ EXPECT_TRUE(table.get_bit(10));
+ table.set_bit(150);
+ EXPECT_TRUE(table.get_bit(150));
+
+ // Test find_first_bit_set_after (strictly after the given index)
+ EXPECT_EQ(table.find_first_bit_set_after(0), size_t(10));
+ EXPECT_EQ(table.find_first_bit_set_after(5), size_t(10));
+ EXPECT_EQ(table.find_first_bit_set_after(9), size_t(10));
+ EXPECT_EQ(table.find_first_bit_set_after(10), size_t(150));
+ EXPECT_EQ(table.find_first_bit_set_after(149), size_t(150));
+ EXPECT_EQ(table.find_first_bit_set_after(150), Table::TOTAL_BITS); // TOTAL_BITS
+ EXPECT_EQ(table.find_first_bit_set_after(151), Table::TOTAL_BITS); // TOTAL_BITS
+
+ // Test clear_bit
+ table.clear_bit(10);
+ EXPECT_FALSE(table.get_bit(10));
+ EXPECT_EQ(table.find_first_bit_set_after(0), size_t(150));
+
+ table.clear_bit(150);
+ EXPECT_FALSE(table.get_bit(150));
+ EXPECT_EQ(table.find_first_bit_set_after(0), Table::TOTAL_BITS);
+}
+
+TEST(LlvmLibcFreeTableTest, InsertAndRemove) {
+ constexpr size_t BITS_PER_ENTRY = LIBC_NAMESPACE::cpp::numeric_limits<uintptr_t>::digits;
+ constexpr size_t NUM_TABLE_ENTRIES = 192 / BITS_PER_ENTRY;
+
+ // 192-bit table
+ using Table = TestTable<32, 3, 2, NUM_TABLE_ENTRIES>;
+ Table table;
+
+ alignas(Block::MIN_ALIGN) LIBC_NAMESPACE::cpp::byte buf[1024];
+ auto result = Block::init(buf);
+ ASSERT_TRUE(result.has_value());
+ Block *block = *result;
+
+ // Make sure block is free
+ block->mark_free();
+
+ // Insert block
+ table.insert(block);
+
+ // Verify that bit is set
+ size_t bit_index = Table::size_to_bit_index(block->inner_size());
+ EXPECT_TRUE(table.get_bit(bit_index));
+
+ // Verify find_first_bit_set_after
+ EXPECT_EQ(table.find_first_bit_set_after(0), bit_index);
+
+ // Remove block
+ table.remove(block);
+
+ // Verify that bit is cleared
+ EXPECT_FALSE(table.get_bit(bit_index));
+ EXPECT_EQ(table.find_first_bit_set_after(0), Table::TOTAL_BITS);
+}
+
+TEST(LlvmLibcFreeTableTest, RemoveFirstFitInList) {
+ constexpr size_t BITS_PER_ENTRY = LIBC_NAMESPACE::cpp::numeric_limits<uintptr_t>::digits;
+ constexpr size_t NUM_TABLE_ENTRIES = 192 / BITS_PER_ENTRY;
+
+ // 192-bit table
+ using Table = TestTable<32, 3, 2, NUM_TABLE_ENTRIES>;
+ Table table;
+
+ alignas(Block::MIN_ALIGN) LIBC_NAMESPACE::cpp::byte buf[4096];
+ auto result = Block::init(buf);
+ ASSERT_TRUE(result.has_value());
+ Block *block = *result;
+ block->mark_free();
+
+ // Split into block1 (size 1024) and the rest
+ auto split_res = block->split(1024);
+ ASSERT_TRUE(split_res.has_value());
+ Block *block1 = block;
+ Block *block2 = *split_res;
+
+ // Split block2 to get size 1120 and the rest
+ auto split_res2 = block2->split(1120);
+ ASSERT_TRUE(split_res2.has_value());
+
+ block1->mark_free();
+ block2->mark_free();
+
+ // Verify that both block1 and block2 map to the same bit index 32
+ size_t bit_index1 = Table::size_to_bit_index(block1->inner_size());
+ size_t bit_index2 = Table::size_to_bit_index(block2->inner_size());
+ EXPECT_EQ(bit_index1, size_t(32));
+ EXPECT_EQ(bit_index2, size_t(32));
+
+ // Insert both blocks into the table
+ table.insert(block1);
+ table.insert(block2);
+
+ // Verify that the bit 32 is set
+ EXPECT_TRUE(table.get_bit(32));
+
+ // Request a size (1050) that only block2 can fit
+ Block *removed = table.remove_first_fit_in_list(32, 1050);
+ EXPECT_EQ(removed, block2);
+
+ // Verify that bit 32 is still set (since block1 is still in the list)
+ EXPECT_TRUE(table.get_bit(32));
+
+ // Request a size (1000) that block1 can fit
+ removed = table.remove_first_fit_in_list(32, 1000);
+ EXPECT_EQ(removed, block1);
+
+ // Verify that bit 32 is now cleared
+ EXPECT_FALSE(table.get_bit(32));
+}
+
+TEST(LlvmLibcFreeTableTest, FindAndRemoveFit) {
+ constexpr size_t BITS_PER_ENTRY = LIBC_NAMESPACE::cpp::numeric_limits<uintptr_t>::digits;
+ constexpr size_t NUM_TABLE_ENTRIES = 192 / BITS_PER_ENTRY;
+
+ // 192-bit table
+ using Table = TestTable<32, 3, 2, NUM_TABLE_ENTRIES>;
+ Table table;
+
+ alignas(Block::MIN_ALIGN) LIBC_NAMESPACE::cpp::byte buf[4096];
+ auto result = Block::init(buf);
+ ASSERT_TRUE(result.has_value());
+ Block *block = *result;
+ block->mark_free();
+
+ // Split to get block1 (1120 B) and block2 (2500 B)
+ auto split_res = block->split(1120);
+ ASSERT_TRUE(split_res.has_value());
+ Block *block1 = block;
+ Block *block2 = *split_res;
+
+ auto split_res2 = block2->split(2500);
+ ASSERT_TRUE(split_res2.has_value());
+
+ block1->mark_free();
+ block2->mark_free();
+
+ // Verify their bit indexes
+ size_t bit_index1 = Table::size_to_bit_index(block1->inner_size());
+ size_t bit_index2 = Table::size_to_bit_index(block2->inner_size());
+ EXPECT_EQ(bit_index1, size_t(32));
+ EXPECT_EQ(bit_index2, size_t(36));
+
+ // 1. Test Exact Fit search path
+ table.insert(block1);
+ EXPECT_TRUE(table.get_bit(32));
+
+ Block *removed = table.find_and_remove_fit(1050);
+ EXPECT_EQ(removed, block1);
+ EXPECT_FALSE(table.get_bit(32));
+
+ // 2. Test Oversized bin (Best-Fit) search path
+ table.insert(block2);
+ EXPECT_TRUE(table.get_bit(36));
+
+ // Request 1050 B (maps to 32). Bin 32 is empty, so it will find and pop the next set bit (36)
+ removed = table.find_and_remove_fit(1050);
+ EXPECT_EQ(removed, block2);
+ EXPECT_FALSE(table.get_bit(36));
+}
diff --git a/libc/test/src/__support/math_extras_test.cpp b/libc/test/src/__support/math_extras_test.cpp
index f5d4dae371827..53a6c2f016b7a 100644
--- a/libc/test/src/__support/math_extras_test.cpp
+++ b/libc/test/src/__support/math_extras_test.cpp
@@ -109,6 +109,19 @@ TYPED_TEST(LlvmLibcBitTest, CountZeros, UnsignedTypesNoBigInt) {
EXPECT_EQ(count_zeros<T>(cpp::numeric_limits<T>::max() >> size_t(i)), i);
}
+TYPED_TEST(LlvmLibcBitTest, FloorILog2, UnsignedTypesNoBigInt) {
+ EXPECT_EQ(floor_ilog2<T>(static_cast<T>(0)), static_cast<T>(0));
+ for (int i = 0; i != cpp::numeric_limits<T>::digits; ++i) {
+ auto val = static_cast<T>(T(1) << size_t(i));
+ EXPECT_EQ(floor_ilog2<T>(val), static_cast<T>(i));
+ if (i > 0) {
+ auto val_plus =
+ static_cast<T>(val | static_cast<T>(T(1) << size_t(i - 1)));
+ EXPECT_EQ(floor_ilog2<T>(val_plus), static_cast<T>(i));
+ }
+ }
+}
+
using UnsignedTypes = testing::TypeList<
#if defined(LIBC_TYPES_HAS_INT128)
__uint128_t,
More information about the libc-commits
mailing list