[libc-commits] [libc] [libc] Implement dual freestore rotation for baremetal heap (PR #209811)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Mon Aug 31 22:31:49 PDT 2026
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/209811
>From 2b350c3d9429c8dbc566fef4325a4935c65b27d8 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Fri, 14 Aug 2026 09:57:39 -0700
Subject: [PATCH 1/3] [libc] Implement dual freestore rotation for baremetal
heap
Implement dual FreeStore rotation in FreeListHeap under LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION.
Allocations and frees operate on active free store. On allocation failure, rotate() flips active store index and migrates/coalesces blocks from previous active store into the new active store.
TAG=agy
CONV=06e3ec11-b213-4c86-9390-882ca141b445
---
.../modules/LLVMLibCCompileOptionRules.cmake | 4 +
libc/config/baremetal/config.json | 5 +
libc/config/config.json | 6 ++
libc/fuzzing/__support/CMakeLists.txt | 2 +
libc/src/__support/block.h | 66 +++++++++++++
libc/src/__support/freelist_heap.h | 96 ++++++++++++++++++-
libc/src/__support/freestore.h | 38 +++++++-
libc/src/__support/freetrie.cpp | 8 ++
libc/src/__support/freetrie.h | 4 +
libc/test/src/__support/block_test.cpp | 12 ++-
.../test/src/__support/freelist_heap_test.cpp | 39 +++++++-
11 files changed, 267 insertions(+), 13 deletions(-)
diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index 99defb24d249a..79f2d80bcba7a 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -189,6 +189,10 @@ function(_get_compile_options_from_config output_var)
libc_add_definition(config_options "LIBC_COPT_PRINTF_DISABLE_BITINT")
endif()
+ if(LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION)
+ libc_add_definition(config_options "LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION")
+ endif()
+
if(LIBC_COPT_USE_C_ASSERT)
list(APPEND config_options "-DLIBC_COPT_USE_C_ASSERT")
endif()
diff --git a/libc/config/baremetal/config.json b/libc/config/baremetal/config.json
index 1c52cd0093e1c..e83ed967771f1 100644
--- a/libc/config/baremetal/config.json
+++ b/libc/config/baremetal/config.json
@@ -75,5 +75,10 @@
"LIBC_CONF_CTYPE_SMALLER_ASCII": {
"value": true
}
+ },
+ "baremetal": {
+ "LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION": {
+ "value": true
+ }
}
}
diff --git a/libc/config/config.json b/libc/config/config.json
index fd7784d3d3e55..f13b5fc9c1ca7 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -189,6 +189,12 @@
"doc": "Trap with SIGFPE when feraiseexcept is called with unmasked floating point exceptions, similar to glibc's behavior. This is currently working only on x86 with SSE."
}
},
+ "baremetal": {
+ "LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION": {
+ "value": true,
+ "doc": "Enable rotational dual freestore in FreeListHeap to delay reuse of freed memory. This, when combined with sanitizers, helps detect Use-After-Free (UAF) bugs more reliably by preventing rapid reallocation of recently freed blocks."
+ }
+ },
"assert": {
"LIBC_COPT_USE_C_ASSERT": {
"value": false,
diff --git a/libc/fuzzing/__support/CMakeLists.txt b/libc/fuzzing/__support/CMakeLists.txt
index be72259036458..99de5cb97dcc5 100644
--- a/libc/fuzzing/__support/CMakeLists.txt
+++ b/libc/fuzzing/__support/CMakeLists.txt
@@ -41,6 +41,8 @@ if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
freelist_heap_fuzz
SRCS
freelist_heap_fuzz.cpp
+ COMPILE_OPTIONS
+ -DLIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
DEPENDS
libc.src.__support.freelist_heap
)
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index be2a71f32a23f..83988070b9b95 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -100,8 +100,17 @@ using cpp::optional;
/// The first block in a list is denoted by having a previous offset of `0`.
class BlockRef {
// Masks for the contents of the next field.
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ static constexpr size_t PREV_FREE_MASK = 0x3; // 2 bits
+ static constexpr size_t LAST_MASK = 1 << 2; // bit 2
+
+ static constexpr size_t PREV_FREE_NONE = 0;
+ static constexpr size_t PREV_FREE_STORE_0 = 1;
+ static constexpr size_t PREV_FREE_STORE_1 = 2;
+#else
static constexpr size_t PREV_FREE_MASK = 1 << 0;
static constexpr size_t LAST_MASK = 1 << 1;
+#endif
static constexpr size_t SIZE_MASK = ~(PREV_FREE_MASK | LAST_MASK);
// Header field offsets. The value at PREV_OFFSET is only meaningful when the
@@ -115,7 +124,11 @@ class BlockRef {
// To ensure block sizes have two lower unused bits, ensure usable space is
// always aligned to at least 4 bytes. (The distances between usable spaces,
// the outer size, is then always also 4-aligned.)
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ static constexpr size_t MIN_ALIGN = cpp::max(size_t{8}, alignof(max_align_t));
+#else
static constexpr size_t MIN_ALIGN = cpp::max(size_t{4}, alignof(max_align_t));
+#endif
LIBC_INLINE constexpr BlockRef() = default;
LIBC_INLINE explicit constexpr BlockRef(cpp::byte *header_ptr)
@@ -232,11 +245,27 @@ class BlockRef {
/// @returns The free block immediately before this one, otherwise null.
LIBC_INLINE BlockRef prev_free() const {
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ if ((load_next() & PREV_FREE_MASK) == PREV_FREE_NONE)
+ return BlockRef();
+#else
if (!(load_next() & PREV_FREE_MASK))
return BlockRef();
+#endif
return BlockRef(nonnull_header_ptr() - load_prev());
}
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ LIBC_INLINE int prev_free_store_index() const {
+ size_t val = (load_next() & PREV_FREE_MASK);
+ if (val == PREV_FREE_STORE_0)
+ return 0;
+ if (val == PREV_FREE_STORE_1)
+ return 1;
+ return -1; // Not free
+ }
+#endif
+
/// @returns Whether the block is unavailable for allocation.
LIBC_INLINE bool used() const { return !next() || !next().prev_free(); }
@@ -248,12 +277,29 @@ class BlockRef {
}
/// Marks this block as free.
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ LIBC_INLINE void mark_free(int store_index) const {
+ LIBC_ASSERT(next() && "last block is always considered used");
+ BlockRef next_block = next();
+ size_t val = 0;
+ if (store_index == 0)
+ val = PREV_FREE_STORE_0;
+ else if (store_index == 1)
+ val = PREV_FREE_STORE_1;
+ LIBC_ASSERT(val != 0 && "Invalid store index");
+
+ size_t next_val = next_block.load_next() & ~PREV_FREE_MASK;
+ next_block.store_next(next_val | val);
+ next_block.store_prev(outer_size());
+ }
+#else
LIBC_INLINE void mark_free() const {
LIBC_ASSERT(next() && "last block is always considered used");
BlockRef next_block = next();
next_block.store_next(next_block.load_next() | PREV_FREE_MASK);
next_block.store_prev(outer_size());
}
+#endif
LIBC_INLINE bool is_usable_space_aligned(size_t alignment) const {
return reinterpret_cast<uintptr_t>(usable_space()) % alignment == 0;
@@ -439,7 +485,11 @@ optional<BlockRef> BlockRef::init(ByteSpan region) {
BlockRef block =
as_block({reinterpret_cast<cpp::byte *>(block_start), last_start_ptr});
make_last_block(last_start_ptr);
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ block.mark_free(0);
+#else
block.mark_free();
+#endif
return block;
}
@@ -460,6 +510,11 @@ BlockRef::BlockInfo BlockRef::allocate(BlockRef block, size_t alignment,
LIBC_ASSERT(maybe_aligned_block.has_value() &&
"it should always be possible to split for alignment");
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ // We skip eager merge here as we cannot tell the store index of original vs
+ // prev, but coalesce_and_insert will check and merge them if appropriate.
+ info.prev = original;
+#else
if (BlockRef prev = original.prev_free()) {
// If there is a free block before this, we can merge the current one with
// the newly created one.
@@ -467,6 +522,7 @@ BlockRef::BlockInfo BlockRef::allocate(BlockRef block, size_t alignment,
} else {
info.prev = original;
}
+#endif
BlockRef aligned_block = *maybe_aligned_block;
LIBC_ASSERT(aligned_block.is_usable_space_aligned(alignment) &&
@@ -505,14 +561,24 @@ optional<BlockRef> BlockRef::split(size_t new_inner_size,
return {};
bool was_free = !used();
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ int orig_store_idx = next().prev_free_store_index();
+ int store_to_set = orig_store_idx >= 0 ? orig_store_idx : 0;
+#endif
ByteSpan new_region = region().subspan(new_outer_size);
store_next((load_next() & ~SIZE_MASK) | new_outer_size);
BlockRef new_block = as_block(new_region);
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ new_block.mark_free(store_to_set);
+ if (was_free)
+ mark_free(store_to_set);
+#else
new_block.mark_free();
if (was_free)
mark_free();
+#endif
LIBC_ASSERT(new_block.is_usable_space_aligned(usable_space_alignment) &&
"usable space must have requested alignment");
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index d2ec9339d72ed..03d41737b372b 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -51,7 +51,14 @@ class FreeListHeap {
void *realloc(void *ptr, size_t size);
void *calloc(size_t num, size_t size);
size_t allocation_size(const void *ptr) const;
- LIBC_INLINE void integrity_check() const { free_store.integrity_check(); }
+ LIBC_INLINE void integrity_check() const {
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ free_stores[0].integrity_check();
+ free_stores[1].integrity_check();
+#else
+ free_store.integrity_check();
+#endif
+ }
cpp::span<cpp::byte> region() const { return {begin, end}; }
@@ -68,15 +75,68 @@ class FreeListHeap {
bool is_valid_ptr(const void *ptr) const { return ptr >= begin && ptr < end; }
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ LIBC_INLINE int get_store_index(BlockRef block) {
+ return block.next().prev_free_store_index();
+ }
+
+ LIBC_INLINE void coalesce_and_insert(BlockRef block, int store_idx) {
+ block.mark_free(store_idx);
+ BlockRef prev = block.prev_free();
+ if (prev) {
+ if (FreeStore::too_small(prev)) {
+ block = prev;
+ block.merge_next();
+ } else {
+ int prev_store = get_store_index(prev);
+ if (prev_store == store_idx) {
+ free_stores[prev_store].remove(prev);
+ block = prev;
+ block.merge_next();
+ }
+ }
+ }
+
+ BlockRef next = block.next();
+ if (!next.used()) {
+ if (FreeStore::too_small(next)) {
+ block.merge_next();
+ } else {
+ int next_store = get_store_index(next);
+ if (next_store == store_idx) {
+ free_stores[next_store].remove(next);
+ block.merge_next();
+ }
+ }
+ }
+ block.mark_free(store_idx);
+ free_stores[store_idx].insert(block);
+ }
+
+ LIBC_INLINE void rotate() {
+ unsigned prev_active = active;
+ active = 1 - active;
+ while (BlockRef block = free_stores[prev_active].remove_any())
+ coalesce_and_insert(block, active);
+ }
+#endif
+
cpp::byte *begin;
cpp::byte *end;
bool is_initialized = false;
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ FreeStore free_stores[2];
+ unsigned active = 0;
+ LIBC_INLINE FreeStore &active_free_store() { return free_stores[active]; }
+#else
FreeStore free_store;
+ LIBC_INLINE FreeStore &active_free_store() { return free_store; }
+#endif
};
template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
public:
- constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
+ LIBC_INLINE constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
private:
cpp::byte buffer[BUFF_SIZE];
@@ -86,8 +146,14 @@ LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ free_stores[0].set_range({0, cpp::bit_ceil(block.inner_size())});
+ free_stores[1].set_range({0, cpp::bit_ceil(block.inner_size())});
+ coalesce_and_insert(block, active);
+#else
free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
free_store.insert(block);
+#endif
is_initialized = true;
}
@@ -102,17 +168,29 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (!request_size)
return nullptr;
- BlockRef block = free_store.remove_best_fit(request_size);
+ BlockRef block = active_free_store().remove_best_fit(request_size);
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ if (!block) {
+ rotate();
+ block = active_free_store().remove_best_fit(request_size);
+ }
+#endif
if (!block)
return nullptr;
auto block_info = BlockRef::allocate(block, alignment, size);
+ block_info.block.mark_used();
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ if (block_info.next)
+ coalesce_and_insert(block_info.next, active);
+ if (block_info.prev)
+ coalesce_and_insert(block_info.prev, active);
+#else
if (block_info.next)
free_store.insert(block_info.next);
if (block_info.prev)
free_store.insert(block_info.prev);
-
- block_info.block.mark_used();
+#endif
return block_info.block.usable_space();
}
@@ -147,6 +225,9 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
BlockRef block = BlockRef::from_usable_space(bytes);
LIBC_ASSERT(block.next() && "sentinel last block cannot be freed");
LIBC_ASSERT(block.used() && "double free");
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ coalesce_and_insert(block, 1 - active);
+#else
block.mark_free();
// Can we combine with the left or right blocks?
@@ -165,6 +246,7 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
}
// Add back to the freelist
free_store.insert(block);
+#endif
}
LIBC_INLINE size_t FreeListHeap::allocation_size(const void *ptr) const {
@@ -186,6 +268,9 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
// register the new block on successful split
if (next.has_value()) {
BlockRef next_block = *next;
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ coalesce_and_insert(next_block, 1 - active);
+#else
BlockRef right = next_block.next();
// Since the original block was not the last block (the sentinel last
// block is never split), the split-off remainder block `next_block` is
@@ -197,6 +282,7 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
next_block.merge_next();
}
free_store.insert(next_block);
+#endif
}
return true;
}
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index ccc4f9555c64a..da440dd022872 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -73,6 +73,40 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
list.integrity_check();
}
+ /// Removes and returns any block from the store.
+ /// @returns The block removed, or BlockRef() if empty.
+ LIBC_INLINE BlockRef remove_any() {
+ for (size_t i = 0; i < TOTAL_BITS - 1; ++i) {
+ if (!free_lists[i].empty()) {
+ BlockRef block = free_lists[i].front();
+ free_lists[i].pop();
+ if (free_lists[i].empty())
+ free_sizes.mark_vacant(i);
+ return block;
+ }
+ }
+ if constexpr (USE_TRIE) {
+ if (BlockRef block = trie.pop_any()) {
+ if (trie.empty())
+ free_sizes.mark_vacant(TOTAL_BITS - 1);
+ return block;
+ }
+ } else {
+ if (!overflow_list.empty()) {
+ BlockRef block = overflow_list.front();
+ overflow_list.pop();
+ if (overflow_list.empty())
+ free_sizes.mark_vacant(TOTAL_BITS - 1);
+ return block;
+ }
+ }
+ return BlockRef();
+ }
+
+ LIBC_INLINE static bool too_small(BlockRef block) {
+ return block.outer_size() < MIN_OUTER_SIZE;
+ }
+
private:
LIBC_INLINE constexpr TLSFFreeStoreImpl(cpp::bool_constant<true>) : trie() {}
LIBC_INLINE constexpr TLSFFreeStoreImpl(cpp::bool_constant<false>)
@@ -100,10 +134,6 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
}
protected:
- LIBC_INLINE static bool too_small(BlockRef block) {
- return block.outer_size() < MIN_OUTER_SIZE;
- }
-
Table free_sizes;
cpp::array<FreeList, TOTAL_BITS - 1> free_lists;
union {
diff --git a/libc/src/__support/freetrie.cpp b/libc/src/__support/freetrie.cpp
index d0392342f2231..f15664db2f12f 100644
--- a/libc/src/__support/freetrie.cpp
+++ b/libc/src/__support/freetrie.cpp
@@ -79,4 +79,12 @@ void FreeTrie::integrity_check() const {
integrity_check_trie_node(integrity_check_trie_node, root());
}
+BlockRef FreeTrie::pop_any() {
+ if (!root_)
+ return BlockRef();
+ Node *node = root_;
+ remove(node);
+ return node->block();
+}
+
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 1be34f456021b..bdb115eeb184b 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -132,6 +132,10 @@ class FreeTrie {
/// Verify integrity of all nodes in the trie.
void integrity_check() const;
+ /// Removes and returns any block from the trie.
+ /// @returns The block removed, or BlockRef() if empty.
+ BlockRef pop_any();
+
private:
/// @returns Whether a node is the head of its containing freelist.
bool is_head(Node *node) const { return node->parent || node == root_; }
diff --git a/libc/test/src/__support/block_test.cpp b/libc/test/src/__support/block_test.cpp
index 6bd0ab5be24c0..053e81b5313e2 100644
--- a/libc/test/src/__support/block_test.cpp
+++ b/libc/test/src/__support/block_test.cpp
@@ -249,7 +249,11 @@ TEST(LlvmLibcBlockTest, CanMarkBlockUsed) {
EXPECT_TRUE(block.used());
EXPECT_EQ(block.outer_size(), orig_size);
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ block.mark_free(0);
+#else
block.mark_free();
+#endif
EXPECT_FALSE(block.used());
}
@@ -469,7 +473,7 @@ TEST(LlvmLibcBlockTest, PreviousBlockMergedIfNotFirst) {
ASSERT_TRUE(result2.has_value());
BlockRef newblock = *result2;
ASSERT_EQ(newblock.prev_free().addr(), block.addr());
- size_t old_prev_size = block.outer_size();
+ [[maybe_unused]] size_t old_prev_size = block.outer_size();
// Now pick an alignment such that the usable space is not already aligned to
// it. We want to explicitly test that the block will split into one before
@@ -481,12 +485,18 @@ TEST(LlvmLibcBlockTest, PreviousBlockMergedIfNotFirst) {
// Ensure we can allocate in the new block.
auto [aligned_block, prev, next] = BlockRef::allocate(newblock, alignment, 1);
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ EXPECT_EQ(prev.addr(), newblock.addr());
+ EXPECT_EQ(aligned_block.prev_free().addr(), newblock.addr());
+ EXPECT_EQ(newblock.next().addr(), aligned_block.addr());
+#else
// Now there should be no new previous block. Instead, the padding we did
// create should be merged into the original previous block.
EXPECT_EQ(prev.addr(), BlockRef().addr());
EXPECT_EQ(aligned_block.prev_free().addr(), block.addr());
EXPECT_EQ(block.next().addr(), aligned_block.addr());
EXPECT_GT(block.outer_size(), old_prev_size);
+#endif
}
TEST(LlvmLibcBlockTest, CanRemergeBlockAllocations) {
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 80b8131174459..3ac02f480e04e 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -97,8 +97,11 @@ TEST_FOR_EACH_ALLOCATOR(CanFreeAndRealloc, 2048) {
void *ptr1 = allocator.allocate(ALLOC_SIZE);
allocator.free(ptr1);
void *ptr2 = allocator.allocate(ALLOC_SIZE);
-
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ EXPECT_NE(ptr1, ptr2);
+#else
EXPECT_EQ(ptr1, ptr2);
+#endif
}
TEST_FOR_EACH_ALLOCATOR(ReturnsNullWhenAllocationTooLarge, 2048) {
@@ -284,7 +287,7 @@ TEST_FOR_EACH_ALLOCATOR(AllocateZero, 2048) {
ASSERT_EQ(ptr, static_cast<void *>(nullptr));
}
-TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 2048) {
+TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 3072) {
constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
@@ -305,7 +308,7 @@ TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 2048) {
// still get aligned allocations even if the underlying buffer is not aligned to
// the alignments we request.
TEST(LlvmLibcFreeListHeap, AlignedAllocUnalignedBuffer) {
- byte buf[4096] = {byte(0)};
+ byte buf[8192] = {byte(0)};
// Ensure the underlying buffer is poorly aligned.
FreeListHeap allocator(span<byte>(buf).subspan(1));
@@ -391,3 +394,33 @@ TEST_FOR_EACH_ALLOCATOR(IntegrityCheck, 2048) {
allocator.free(ptr2);
allocator.integrity_check();
}
+
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+TEST(LlvmLibcFreeListHeap, RotationSmokeTest) {
+ byte buf[4096] = {byte(0)};
+ FreeListHeap allocator(buf);
+
+ constexpr size_t SIZES[] = {64, 128, 256, 512};
+
+ for (size_t size : SIZES) {
+ void *ptr1 = allocator.allocate(size);
+ ASSERT_NE(ptr1, static_cast<void *>(nullptr));
+ allocator.free(ptr1);
+
+ // Because free() quarantines blocks into non-active store (1 - active),
+ // allocating again from active store yields a different address even for
+ // the same requested size.
+ void *ptr2 = allocator.allocate(size);
+ ASSERT_NE(ptr2, static_cast<void *>(nullptr));
+ EXPECT_NE(ptr1, ptr2);
+ allocator.free(ptr2);
+ }
+
+ // Trigger store rotation by requesting a size larger than remaining free
+ // space in active store, forcing migration and coalescing of all quarantined
+ // blocks into active store.
+ void *large_ptr = allocator.allocate(3500);
+ ASSERT_NE(large_ptr, static_cast<void *>(nullptr));
+ allocator.free(large_ptr);
+}
+#endif
>From ddce7d96731254cc3aedf0c6cdd220b214b8d3c3 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Fri, 28 Aug 2026 10:47:19 -0400
Subject: [PATCH 2/3] [libc][baremetal] address CR and try to unify rotation
with normal setup
---
libc/config/baremetal/config.json | 5 -
libc/config/config.json | 4 +-
libc/fuzzing/__support/CMakeLists.txt | 12 ++
libc/src/__support/CMakeLists.txt | 1 +
libc/src/__support/block.h | 117 +++++-------
libc/src/__support/freelist_heap.h | 178 ++++++++----------
libc/test/src/__support/CMakeLists.txt | 18 ++
libc/test/src/__support/block_test.cpp | 12 +-
.../test/src/__support/freelist_heap_test.cpp | 61 +++---
9 files changed, 186 insertions(+), 222 deletions(-)
diff --git a/libc/config/baremetal/config.json b/libc/config/baremetal/config.json
index e83ed967771f1..1c52cd0093e1c 100644
--- a/libc/config/baremetal/config.json
+++ b/libc/config/baremetal/config.json
@@ -75,10 +75,5 @@
"LIBC_CONF_CTYPE_SMALLER_ASCII": {
"value": true
}
- },
- "baremetal": {
- "LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION": {
- "value": true
- }
}
}
diff --git a/libc/config/config.json b/libc/config/config.json
index f13b5fc9c1ca7..604b03c9b5688 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -191,8 +191,8 @@
},
"baremetal": {
"LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION": {
- "value": true,
- "doc": "Enable rotational dual freestore in FreeListHeap to delay reuse of freed memory. This, when combined with sanitizers, helps detect Use-After-Free (UAF) bugs more reliably by preventing rapid reallocation of recently freed blocks."
+ "value": false,
+ "doc": "Enable rotational dual freestore in FreeListHeap to delay reuse of freed memory. This, when combined with sanitizers, helps detect Use-After-Free (UAF) bugs more reliably by preventing rapid reallocation of recently freed blocks. It costs one extra flag bit in the block header, which doubles the minimum allocation alignment on 32-bit targets."
}
},
"assert": {
diff --git a/libc/fuzzing/__support/CMakeLists.txt b/libc/fuzzing/__support/CMakeLists.txt
index 99de5cb97dcc5..2fbaa2f7cd6cf 100644
--- a/libc/fuzzing/__support/CMakeLists.txt
+++ b/libc/fuzzing/__support/CMakeLists.txt
@@ -39,6 +39,15 @@ add_libc_fuzzer(
if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
add_libc_fuzzer(
freelist_heap_fuzz
+ SRCS
+ freelist_heap_fuzz.cpp
+ DEPENDS
+ libc.src.__support.freelist_heap
+ )
+ # Same fuzzer, but with the heap rotating between several free stores, so
+ # that both configurations get covered by a single build.
+ add_libc_fuzzer(
+ freelist_heap_rotation_fuzz
SRCS
freelist_heap_fuzz.cpp
COMPILE_OPTIONS
@@ -49,8 +58,11 @@ if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
# TODO(#119995): Remove this once sccache on Windows no longer requires
# the use of -DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded.
get_fq_target_name(freelist_heap_fuzz freelist_heap_fuzz_target_name)
+ get_fq_target_name(freelist_heap_rotation_fuzz
+ freelist_heap_rotation_fuzz_target_name)
set_target_properties(
${freelist_heap_fuzz_target_name}
+ ${freelist_heap_rotation_fuzz_target_name}
PROPERTIES
MSVC_DEBUG_INFORMATION_FORMAT ""
)
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index d3fd8a060ab15..16b2f21cae8a8 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -17,6 +17,7 @@ add_header_library(
DEPENDS
libc.hdr.stdint_proxy
libc.src.__support.CPP.algorithm
+ libc.src.__support.CPP.bit
libc.src.__support.CPP.limits
libc.src.__support.CPP.new
libc.src.__support.CPP.optional
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index 83988070b9b95..d002dae2c1621 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -17,6 +17,7 @@
#include "hdr/stdint_proxy.h"
#include "hdr/types/size_t.h"
#include "src/__support/CPP/algorithm.h"
+#include "src/__support/CPP/bit.h"
#include "src/__support/CPP/cstddef.h"
#include "src/__support/CPP/limits.h"
#include "src/__support/CPP/new.h"
@@ -99,18 +100,22 @@ using cpp::optional;
/// The next offset of a block matches the previous offset of its next block.
/// The first block in a list is denoted by having a previous offset of `0`.
class BlockRef {
- // Masks for the contents of the next field.
+public:
+ /// The number of free stores that free blocks are distributed over.
+ ///
+ /// Every free block records which store owns it, which lets an allocator
+ /// hold several stores and rotate between them; see FreeListHeap. Each
+ /// additional store widens the free field below, and hence MIN_ALIGN.
#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- static constexpr size_t PREV_FREE_MASK = 0x3; // 2 bits
- static constexpr size_t LAST_MASK = 1 << 2; // bit 2
-
- static constexpr size_t PREV_FREE_NONE = 0;
- static constexpr size_t PREV_FREE_STORE_0 = 1;
- static constexpr size_t PREV_FREE_STORE_1 = 2;
+ static constexpr size_t NUM_FREE_STORES = 2;
#else
- static constexpr size_t PREV_FREE_MASK = 1 << 0;
- static constexpr size_t LAST_MASK = 1 << 1;
+ static constexpr size_t NUM_FREE_STORES = 1;
#endif
+
+private:
+ static constexpr size_t PREV_FREE_BITS = cpp::bit_width(NUM_FREE_STORES);
+ static constexpr size_t PREV_FREE_MASK = (size_t{1} << PREV_FREE_BITS) - 1;
+ static constexpr size_t LAST_MASK = size_t{1} << PREV_FREE_BITS;
static constexpr size_t SIZE_MASK = ~(PREV_FREE_MASK | LAST_MASK);
// Header field offsets. The value at PREV_OFFSET is only meaningful when the
@@ -121,14 +126,10 @@ class BlockRef {
public:
static constexpr size_t HEADER_SIZE = NEXT_OFFSET + sizeof(size_t);
- // To ensure block sizes have two lower unused bits, ensure usable space is
- // always aligned to at least 4 bytes. (The distances between usable spaces,
- // the outer size, is then always also 4-aligned.)
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- static constexpr size_t MIN_ALIGN = cpp::max(size_t{8}, alignof(max_align_t));
-#else
- static constexpr size_t MIN_ALIGN = cpp::max(size_t{4}, alignof(max_align_t));
-#endif
+ // The flag bits above live in the lower bits of a block's size, so the
+ // usable space must be aligned to the first bit they leave unused.
+ static constexpr size_t MIN_ALIGN =
+ cpp::max(LAST_MASK << 1, alignof(max_align_t));
LIBC_INLINE constexpr BlockRef() = default;
LIBC_INLINE explicit constexpr BlockRef(cpp::byte *header_ptr)
@@ -245,26 +246,16 @@ class BlockRef {
/// @returns The free block immediately before this one, otherwise null.
LIBC_INLINE BlockRef prev_free() const {
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- if ((load_next() & PREV_FREE_MASK) == PREV_FREE_NONE)
- return BlockRef();
-#else
if (!(load_next() & PREV_FREE_MASK))
return BlockRef();
-#endif
return BlockRef(nonnull_header_ptr() - load_prev());
}
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ /// @returns The index of the free store owning the block immediately before
+ /// this one, or a negative value if that block is not free.
LIBC_INLINE int prev_free_store_index() const {
- size_t val = (load_next() & PREV_FREE_MASK);
- if (val == PREV_FREE_STORE_0)
- return 0;
- if (val == PREV_FREE_STORE_1)
- return 1;
- return -1; // Not free
+ return static_cast<int>(load_next() & PREV_FREE_MASK) - 1;
}
-#endif
/// @returns Whether the block is unavailable for allocation.
LIBC_INLINE bool used() const { return !next() || !next().prev_free(); }
@@ -276,30 +267,16 @@ class BlockRef {
next_block.store_next(next_block.load_next() & ~PREV_FREE_MASK);
}
- /// Marks this block as free.
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- LIBC_INLINE void mark_free(int store_index) const {
- LIBC_ASSERT(next() && "last block is always considered used");
- BlockRef next_block = next();
- size_t val = 0;
- if (store_index == 0)
- val = PREV_FREE_STORE_0;
- else if (store_index == 1)
- val = PREV_FREE_STORE_1;
- LIBC_ASSERT(val != 0 && "Invalid store index");
-
- size_t next_val = next_block.load_next() & ~PREV_FREE_MASK;
- next_block.store_next(next_val | val);
- next_block.store_prev(outer_size());
- }
-#else
- LIBC_INLINE void mark_free() const {
+ /// Marks this block as free and owned by the given free store.
+ LIBC_INLINE void mark_free(size_t store_index = 0) const {
LIBC_ASSERT(next() && "last block is always considered used");
+ LIBC_ASSERT(store_index < NUM_FREE_STORES && "invalid free store index");
BlockRef next_block = next();
- next_block.store_next(next_block.load_next() | PREV_FREE_MASK);
+ // Replace the free field with `store_index + 1`, as 0 encodes "in use".
+ next_block.store_next((next_block.load_next() & ~PREV_FREE_MASK) |
+ (store_index + 1));
next_block.store_prev(outer_size());
}
-#endif
LIBC_INLINE bool is_usable_space_aligned(size_t alignment) const {
return reinterpret_cast<uintptr_t>(usable_space()) % alignment == 0;
@@ -485,11 +462,7 @@ optional<BlockRef> BlockRef::init(ByteSpan region) {
BlockRef block =
as_block({reinterpret_cast<cpp::byte *>(block_start), last_start_ptr});
make_last_block(last_start_ptr);
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- block.mark_free(0);
-#else
block.mark_free();
-#endif
return block;
}
@@ -510,19 +483,17 @@ BlockRef::BlockInfo BlockRef::allocate(BlockRef block, size_t alignment,
LIBC_ASSERT(maybe_aligned_block.has_value() &&
"it should always be possible to split for alignment");
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- // We skip eager merge here as we cannot tell the store index of original vs
- // prev, but coalesce_and_insert will check and merge them if appropriate.
- info.prev = original;
-#else
- if (BlockRef prev = original.prev_free()) {
- // If there is a free block before this, we can merge the current one with
- // the newly created one.
+ // If the block before the padding is free and owned by the same free store,
+ // the two can be merged; blocks owned by different stores must be kept
+ // apart. Otherwise the padding is handed back to the caller, which decides
+ // which store it belongs to.
+ BlockRef prev = original.prev_free();
+ if (prev && original.prev_free_store_index() ==
+ original.next().prev_free_store_index()) {
prev.merge_next();
} else {
info.prev = original;
}
-#endif
BlockRef aligned_block = *maybe_aligned_block;
LIBC_ASSERT(aligned_block.is_usable_space_aligned(alignment) &&
@@ -561,24 +532,20 @@ optional<BlockRef> BlockRef::split(size_t new_inner_size,
return {};
bool was_free = !used();
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- int orig_store_idx = next().prev_free_store_index();
- int store_to_set = orig_store_idx >= 0 ? orig_store_idx : 0;
-#endif
+ // The block split off below is always free. If this block is free too, both
+ // halves must stay in the same store to remain mergeable; if it is in use,
+ // the caller owns the new block and will move it to whichever store it
+ // wants, so any valid index will do.
+ size_t store_index =
+ was_free ? static_cast<size_t>(next().prev_free_store_index()) : 0;
ByteSpan new_region = region().subspan(new_outer_size);
store_next((load_next() & ~SIZE_MASK) | new_outer_size);
BlockRef new_block = as_block(new_region);
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- new_block.mark_free(store_to_set);
- if (was_free)
- mark_free(store_to_set);
-#else
- new_block.mark_free();
+ new_block.mark_free(store_index);
if (was_free)
- mark_free();
-#endif
+ mark_free(store_index);
LIBC_ASSERT(new_block.is_usable_space_aligned(usable_space_alignment) &&
"usable space must have requested alignment");
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 03d41737b372b..8487d44846244 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -38,6 +38,18 @@ LIBC_INLINE constexpr bool IsPow2(size_t x) { return x && (x & (x - 1)) == 0; }
class FreeListHeap {
public:
+ /// The heap keeps its free blocks in NUM_FREE_STORES stores and rotates
+ /// between them: allocations are served from the active store, while frees
+ /// go to the next one, quarantining the memory there. Only once the active
+ /// store cannot satisfy a request does rotate() make the quarantined memory
+ /// available again. Delaying reuse this way makes use-after-free bugs much
+ /// more likely to be caught, e.g. by a sanitizer.
+ ///
+ /// With a single store (the default) the quarantine store is the active
+ /// store, no rotation ever happens, and this degenerates into the usual
+ /// immediate-reuse behavior.
+ static constexpr size_t NUM_FREE_STORES = BlockRef::NUM_FREE_STORES;
+
constexpr FreeListHeap() : begin(&_end), end(&__llvm_libc_heap_limit) {}
constexpr FreeListHeap(span<cpp::byte> region)
@@ -52,12 +64,8 @@ class FreeListHeap {
void *calloc(size_t num, size_t size);
size_t allocation_size(const void *ptr) const;
LIBC_INLINE void integrity_check() const {
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- free_stores[0].integrity_check();
- free_stores[1].integrity_check();
-#else
- free_store.integrity_check();
-#endif
+ for (const FreeStore &store : free_stores)
+ store.integrity_check();
}
cpp::span<cpp::byte> region() const { return {begin, end}; }
@@ -75,63 +83,74 @@ class FreeListHeap {
bool is_valid_ptr(const void *ptr) const { return ptr >= begin && ptr < end; }
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- LIBC_INLINE int get_store_index(BlockRef block) {
- return block.next().prev_free_store_index();
+ /// The store that allocations are served from.
+ LIBC_INLINE FreeStore &active_free_store() { return free_stores[active]; }
+
+ /// @returns The index of the store that receives newly freed blocks. With a
+ /// single free store this is the active store itself, so freed memory is
+ /// immediately available again.
+ LIBC_INLINE size_t quarantine_store_index() const {
+ return (active + 1) % NUM_FREE_STORES;
}
- LIBC_INLINE void coalesce_and_insert(BlockRef block, int store_idx) {
- block.mark_free(store_idx);
+ /// @returns Whether `neighbor`, a free block adjacent to a block owned by
+ /// `store_index`, can be merged into it.
+ LIBC_INLINE static bool can_merge(BlockRef neighbor, size_t store_index) {
+ // Blocks too small to be tracked are owned by no store, so they can always
+ // be absorbed. Anything else must belong to the same store: merging across
+ // stores would hand memory quarantined in an inactive store back out
+ // through the active one.
+ return FreeStore::too_small(neighbor) ||
+ neighbor.next().prev_free_store_index() ==
+ static_cast<int>(store_index);
+ }
+
+ /// Marks `block` free, coalesces it with the adjacent free blocks that may
+ /// be merged into store `store_index`, and inserts the result into that
+ /// store.
+ LIBC_INLINE void coalesce_and_insert(BlockRef block, size_t store_index) {
+ block.mark_free(store_index);
+
BlockRef prev = block.prev_free();
- if (prev) {
- if (FreeStore::too_small(prev)) {
- block = prev;
- block.merge_next();
- } else {
- int prev_store = get_store_index(prev);
- if (prev_store == store_idx) {
- free_stores[prev_store].remove(prev);
- block = prev;
- block.merge_next();
- }
- }
+ if (prev && can_merge(prev, store_index)) {
+ // Removing a block too small to be tracked is a no-op.
+ free_stores[store_index].remove(prev);
+ block = prev;
+ block.merge_next();
}
BlockRef next = block.next();
- if (!next.used()) {
- if (FreeStore::too_small(next)) {
- block.merge_next();
- } else {
- int next_store = get_store_index(next);
- if (next_store == store_idx) {
- free_stores[next_store].remove(next);
- block.merge_next();
- }
- }
+ if (!next.used() && can_merge(next, store_index)) {
+ free_stores[store_index].remove(next);
+ block.merge_next();
}
- block.mark_free(store_idx);
- free_stores[store_idx].insert(block);
+
+ // Merging moved the block boundaries, and an absorbed block may have been
+ // owned by another store, so record the ownership of the result again.
+ block.mark_free(store_index);
+ free_stores[store_index].insert(block);
}
+ /// Ends the current quarantine period: the store that has been collecting
+ /// freed blocks becomes the active one, and whatever is left in the
+ /// previously active store is migrated (and coalesced) into it, so that the
+ /// newly active store holds all the free memory of the heap.
LIBC_INLINE void rotate() {
- unsigned prev_active = active;
- active = 1 - active;
+ // Nothing to rotate to; blocks are never quarantined in the first place.
+ if (NUM_FREE_STORES < 2)
+ return;
+
+ size_t prev_active = active;
+ active = quarantine_store_index();
while (BlockRef block = free_stores[prev_active].remove_any())
coalesce_and_insert(block, active);
}
-#endif
cpp::byte *begin;
cpp::byte *end;
bool is_initialized = false;
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- FreeStore free_stores[2];
- unsigned active = 0;
- LIBC_INLINE FreeStore &active_free_store() { return free_stores[active]; }
-#else
- FreeStore free_store;
- LIBC_INLINE FreeStore &active_free_store() { return free_store; }
-#endif
+ FreeStore free_stores[NUM_FREE_STORES];
+ size_t active = 0;
};
template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
@@ -146,14 +165,9 @@ LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- free_stores[0].set_range({0, cpp::bit_ceil(block.inner_size())});
- free_stores[1].set_range({0, cpp::bit_ceil(block.inner_size())});
- coalesce_and_insert(block, active);
-#else
- free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
- free_store.insert(block);
-#endif
+ for (FreeStore &store : free_stores)
+ store.set_range({0, cpp::bit_ceil(block.inner_size())});
+ free_stores[active].insert(block);
is_initialized = true;
}
@@ -169,28 +183,23 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
return nullptr;
BlockRef block = active_free_store().remove_best_fit(request_size);
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- if (!block) {
+ if (!block && NUM_FREE_STORES > 1) {
+ // The active store is out of memory; make the quarantined blocks available
+ // again and retry.
rotate();
block = active_free_store().remove_best_fit(request_size);
}
-#endif
if (!block)
return nullptr;
auto block_info = BlockRef::allocate(block, alignment, size);
block_info.block.mark_used();
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ // The leftovers of the block were never handed out, so they stay in the
+ // active store rather than being quarantined.
if (block_info.next)
coalesce_and_insert(block_info.next, active);
if (block_info.prev)
coalesce_and_insert(block_info.prev, active);
-#else
- if (block_info.next)
- free_store.insert(block_info.next);
- if (block_info.prev)
- free_store.insert(block_info.prev);
-#endif
return block_info.block.usable_space();
}
@@ -225,28 +234,7 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
BlockRef block = BlockRef::from_usable_space(bytes);
LIBC_ASSERT(block.next() && "sentinel last block cannot be freed");
LIBC_ASSERT(block.used() && "double free");
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- coalesce_and_insert(block, 1 - active);
-#else
- block.mark_free();
-
- // Can we combine with the left or right blocks?
- BlockRef prev_free = block.prev_free();
- BlockRef next = block.next();
-
- if (prev_free) {
- // Remove from free store and merge.
- free_store.remove(prev_free);
- block = prev_free;
- block.merge_next();
- }
- if (!next.used()) {
- free_store.remove(next);
- block.merge_next();
- }
- // Add back to the freelist
- free_store.insert(block);
-#endif
+ coalesce_and_insert(block, quarantine_store_index());
}
LIBC_INLINE size_t FreeListHeap::allocation_size(const void *ptr) const {
@@ -268,21 +256,13 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
// register the new block on successful split
if (next.has_value()) {
BlockRef next_block = *next;
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- coalesce_and_insert(next_block, 1 - active);
-#else
- BlockRef right = next_block.next();
// Since the original block was not the last block (the sentinel last
// block is never split), the split-off remainder block `next_block` is
- // also not the last block. Thus, its next block `right` is guaranteed
- // to be non-null.
- LIBC_ASSERT(right && "right block must be non-null");
- if (!right.used()) {
- free_store.remove(right);
- next_block.merge_next();
- }
- free_store.insert(next_block);
-#endif
+ // also not the last block. Thus, its next block is guaranteed to be
+ // non-null.
+ LIBC_ASSERT(next_block.next() && "right block must be non-null");
+ // The remainder is memory the caller has given up, so quarantine it.
+ coalesce_and_insert(next_block, quarantine_store_index());
}
return true;
}
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 4aca55b8904e1..7582d65f9d722 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -82,6 +82,24 @@ if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
libc.src.string.memory_utils.inline_memset
)
+ # Same tests, but with the heap rotating between several free stores, so that
+ # both configurations get covered by a single build.
+ add_libc_test(
+ freelist_heap_rotation_test
+ SUITE
+ libc-support-tests
+ SRCS
+ freelist_heap_test.cpp
+ COMPILE_OPTIONS
+ -DLIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ DEPENDS
+ libc.src.__support.CPP.span
+ libc.src.__support.freelist_heap
+ libc.src.string.memcmp
+ libc.src.string.memcpy
+ libc.src.string.memory_utils.inline_memset
+ )
+
add_libc_test(
freelist_heap_death_test
SUITE
diff --git a/libc/test/src/__support/block_test.cpp b/libc/test/src/__support/block_test.cpp
index 053e81b5313e2..6bd0ab5be24c0 100644
--- a/libc/test/src/__support/block_test.cpp
+++ b/libc/test/src/__support/block_test.cpp
@@ -249,11 +249,7 @@ TEST(LlvmLibcBlockTest, CanMarkBlockUsed) {
EXPECT_TRUE(block.used());
EXPECT_EQ(block.outer_size(), orig_size);
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- block.mark_free(0);
-#else
block.mark_free();
-#endif
EXPECT_FALSE(block.used());
}
@@ -473,7 +469,7 @@ TEST(LlvmLibcBlockTest, PreviousBlockMergedIfNotFirst) {
ASSERT_TRUE(result2.has_value());
BlockRef newblock = *result2;
ASSERT_EQ(newblock.prev_free().addr(), block.addr());
- [[maybe_unused]] size_t old_prev_size = block.outer_size();
+ size_t old_prev_size = block.outer_size();
// Now pick an alignment such that the usable space is not already aligned to
// it. We want to explicitly test that the block will split into one before
@@ -485,18 +481,12 @@ TEST(LlvmLibcBlockTest, PreviousBlockMergedIfNotFirst) {
// Ensure we can allocate in the new block.
auto [aligned_block, prev, next] = BlockRef::allocate(newblock, alignment, 1);
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- EXPECT_EQ(prev.addr(), newblock.addr());
- EXPECT_EQ(aligned_block.prev_free().addr(), newblock.addr());
- EXPECT_EQ(newblock.next().addr(), aligned_block.addr());
-#else
// Now there should be no new previous block. Instead, the padding we did
// create should be merged into the original previous block.
EXPECT_EQ(prev.addr(), BlockRef().addr());
EXPECT_EQ(aligned_block.prev_free().addr(), block.addr());
EXPECT_EQ(block.next().addr(), aligned_block.addr());
EXPECT_GT(block.outer_size(), old_prev_size);
-#endif
}
TEST(LlvmLibcBlockTest, CanRemergeBlockAllocations) {
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 3ac02f480e04e..23178adfb27bf 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -97,11 +97,12 @@ TEST_FOR_EACH_ALLOCATOR(CanFreeAndRealloc, 2048) {
void *ptr1 = allocator.allocate(ALLOC_SIZE);
allocator.free(ptr1);
void *ptr2 = allocator.allocate(ALLOC_SIZE);
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
- EXPECT_NE(ptr1, ptr2);
-#else
- EXPECT_EQ(ptr1, ptr2);
-#endif
+ // The freed block is quarantined in the inactive free store, so it cannot
+ // be handed back out until the stores rotate.
+ if constexpr (FreeListHeap::NUM_FREE_STORES > 1)
+ EXPECT_NE(ptr1, ptr2);
+ else
+ EXPECT_EQ(ptr1, ptr2);
}
TEST_FOR_EACH_ALLOCATOR(ReturnsNullWhenAllocationTooLarge, 2048) {
@@ -395,32 +396,32 @@ TEST_FOR_EACH_ALLOCATOR(IntegrityCheck, 2048) {
allocator.integrity_check();
}
-#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
TEST(LlvmLibcFreeListHeap, RotationSmokeTest) {
- byte buf[4096] = {byte(0)};
- FreeListHeap allocator(buf);
-
- constexpr size_t SIZES[] = {64, 128, 256, 512};
-
- for (size_t size : SIZES) {
- void *ptr1 = allocator.allocate(size);
- ASSERT_NE(ptr1, static_cast<void *>(nullptr));
- allocator.free(ptr1);
+ if constexpr (FreeListHeap::NUM_FREE_STORES > 1) {
+ byte buf[4096] = {byte(0)};
+ FreeListHeap allocator(buf);
+
+ constexpr size_t SIZES[] = {64, 128, 256, 512};
+
+ for (size_t size : SIZES) {
+ void *ptr1 = allocator.allocate(size);
+ ASSERT_NE(ptr1, static_cast<void *>(nullptr));
+ allocator.free(ptr1);
+
+ // free() quarantines the block in the inactive store, so allocating the
+ // same size again yields a different address.
+ void *ptr2 = allocator.allocate(size);
+ ASSERT_NE(ptr2, static_cast<void *>(nullptr));
+ EXPECT_NE(ptr1, ptr2);
+ allocator.free(ptr2);
+ }
- // Because free() quarantines blocks into non-active store (1 - active),
- // allocating again from active store yields a different address even for
- // the same requested size.
- void *ptr2 = allocator.allocate(size);
- ASSERT_NE(ptr2, static_cast<void *>(nullptr));
- EXPECT_NE(ptr1, ptr2);
- allocator.free(ptr2);
+ // Ask for more than the active store has left, which forces a rotation:
+ // all the quarantined blocks are migrated and coalesced into the store
+ // that is about to become active.
+ void *large_ptr = allocator.allocate(3500);
+ ASSERT_NE(large_ptr, static_cast<void *>(nullptr));
+ allocator.integrity_check();
+ allocator.free(large_ptr);
}
-
- // Trigger store rotation by requesting a size larger than remaining free
- // space in active store, forcing migration and coalescing of all quarantined
- // blocks into active store.
- void *large_ptr = allocator.allocate(3500);
- ASSERT_NE(large_ptr, static_cast<void *>(nullptr));
- allocator.free(large_ptr);
}
-#endif
>From e3cf1851e1440b1ba084934237741c8b1f61e829 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Tue, 1 Sep 2026 01:31:31 -0400
Subject: [PATCH 3/3] fix fmt
---
libc/test/src/__support/freelist_heap_test.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 23178adfb27bf..92b5c76c200d3 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -99,7 +99,7 @@ TEST_FOR_EACH_ALLOCATOR(CanFreeAndRealloc, 2048) {
void *ptr2 = allocator.allocate(ALLOC_SIZE);
// The freed block is quarantined in the inactive free store, so it cannot
// be handed back out until the stores rotate.
- if constexpr (FreeListHeap::NUM_FREE_STORES > 1)
+ if constexpr (FreeListHeap::NUM_FREE_STORES > 1)
EXPECT_NE(ptr1, ptr2);
else
EXPECT_EQ(ptr1, ptr2);
More information about the libc-commits
mailing list