[libc-commits] [libc] [libc] Implement dual freestore rotation for baremetal heap (PR #209811)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Fri Jul 24 12:59:32 PDT 2026
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/209811
>From 6751656485a5e14cbcc0040c56f3c4bf6cea6219 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Tue, 14 Jul 2026 10:23:52 -0700
Subject: [PATCH 1/3] Introduce
LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION option
This option enables rotational dual freestore in FreeListHeap to delay
reuse of freed memory. This, when combined with sanitizers, helps detect
Use-After-Free (UAF) bugs more reliably.
TAG=agy
CONV=06e3ec11-b213-4c86-9390-882ca141b445
---
libc/cmake/modules/LLVMLibCCompileOptionRules.cmake | 4 ++++
libc/config/baremetal/config.json | 5 +++++
libc/config/config.json | 6 ++++++
3 files changed, 15 insertions(+)
diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index 600df74f552b0..95969371c6014 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -189,6 +189,10 @@ function(_get_compile_options_from_config output_var)
libc_add_definition(config_options "LIBC_COPT_PRINTF_DISABLE_BITINT")
endif()
+ if(LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION)
+ libc_add_definition(config_options "LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION")
+ endif()
+
if(LIBC_COPT_USE_C_ASSERT)
list(APPEND config_options "-DLIBC_COPT_USE_C_ASSERT")
endif()
diff --git a/libc/config/baremetal/config.json b/libc/config/baremetal/config.json
index 1c52cd0093e1c..e83ed967771f1 100644
--- a/libc/config/baremetal/config.json
+++ b/libc/config/baremetal/config.json
@@ -75,5 +75,10 @@
"LIBC_CONF_CTYPE_SMALLER_ASCII": {
"value": true
}
+ },
+ "baremetal": {
+ "LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION": {
+ "value": true
+ }
}
}
diff --git a/libc/config/config.json b/libc/config/config.json
index d576d7d14e4af..ebd949f3e6caf 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -185,6 +185,12 @@
"doc": "Trap with SIGFPE when feraiseexcept is called with unmasked floating point exceptions, similar to glibc's behavior. This is currently working only on x86 with SSE."
}
},
+ "baremetal": {
+ "LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION": {
+ "value": false,
+ "doc": "Enable rotational dual freestore in FreeListHeap to delay reuse of freed memory. This, when combined with sanitizers, helps detect Use-After-Free (UAF) bugs more reliably by preventing rapid reallocation of recently freed blocks."
+ }
+ },
"assert": {
"LIBC_COPT_USE_C_ASSERT": {
"value": false,
>From 6c10ca0002622f91a2d2a8ba24cb595189d1929d Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Tue, 14 Jul 2026 10:56:22 -0700
Subject: [PATCH 2/3] Change prev_free to 2 bits to support dual freestore
index
When LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION is enabled,
change BlockRef's prev_free flag to 2 bits to store the index of the
freestore the block belongs to. Add prev_free_store_index() to query the
index and update mark_free() to set the correct index.
TAG=agy
CONV=06e3ec11-b213-4c86-9390-882ca141b445
---
libc/src/__support/block.h | 43 ++++++++++++++++++++++++++++++++++++++
1 file changed, 43 insertions(+)
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index be2a71f32a23f..5154379939c1e 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -100,8 +100,17 @@ using cpp::optional;
/// The first block in a list is denoted by having a previous offset of `0`.
class BlockRef {
// Masks for the contents of the next field.
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ static constexpr size_t PREV_FREE_MASK = 0x3; // 2 bits
+ static constexpr size_t LAST_MASK = 1 << 2; // bit 2
+
+ static constexpr size_t PREV_FREE_NONE = 0;
+ static constexpr size_t PREV_FREE_STORE_0 = 1;
+ static constexpr size_t PREV_FREE_STORE_1 = 2;
+#else
static constexpr size_t PREV_FREE_MASK = 1 << 0;
static constexpr size_t LAST_MASK = 1 << 1;
+#endif
static constexpr size_t SIZE_MASK = ~(PREV_FREE_MASK | LAST_MASK);
// Header field offsets. The value at PREV_OFFSET is only meaningful when the
@@ -115,7 +124,11 @@ class BlockRef {
// To ensure block sizes have two lower unused bits, ensure usable space is
// always aligned to at least 4 bytes. (The distances between usable spaces,
// the outer size, is then always also 4-aligned.)
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ static constexpr size_t MIN_ALIGN = cpp::max(size_t{8}, alignof(max_align_t));
+#else
static constexpr size_t MIN_ALIGN = cpp::max(size_t{4}, alignof(max_align_t));
+#endif
LIBC_INLINE constexpr BlockRef() = default;
LIBC_INLINE explicit constexpr BlockRef(cpp::byte *header_ptr)
@@ -232,11 +245,25 @@ class BlockRef {
/// @returns The free block immediately before this one, otherwise null.
LIBC_INLINE BlockRef prev_free() const {
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ if ((load_next() & PREV_FREE_MASK) == PREV_FREE_NONE)
+ return BlockRef();
+#else
if (!(load_next() & PREV_FREE_MASK))
return BlockRef();
+#endif
return BlockRef(nonnull_header_ptr() - load_prev());
}
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ LIBC_INLINE int prev_free_store_index() const {
+ size_t val = (load_next() & PREV_FREE_MASK);
+ if (val == PREV_FREE_STORE_0) return 0;
+ if (val == PREV_FREE_STORE_1) return 1;
+ return -1; // Not free
+ }
+#endif
+
/// @returns Whether the block is unavailable for allocation.
LIBC_INLINE bool used() const { return !next() || !next().prev_free(); }
@@ -248,12 +275,28 @@ class BlockRef {
}
/// Marks this block as free.
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ LIBC_INLINE void mark_free(int store_index) const {
+ LIBC_ASSERT(next() && "last block is always considered used");
+ BlockRef next_block = next();
+ size_t val = 0;
+ if (store_index == 0) val = PREV_FREE_STORE_0;
+ else if (store_index == 1) val = PREV_FREE_STORE_1;
+ LIBC_ASSERT(val != 0 && "Invalid store index");
+
+ size_t next_val = next_block.load_next() & ~PREV_FREE_MASK;
+ next_block.store_next(next_val | val);
+ next_block.store_prev(outer_size());
+ }
+ LIBC_INLINE void mark_free() const { mark_free(0); }
+#else
LIBC_INLINE void mark_free() const {
LIBC_ASSERT(next() && "last block is always considered used");
BlockRef next_block = next();
next_block.store_next(next_block.load_next() | PREV_FREE_MASK);
next_block.store_prev(outer_size());
}
+#endif
LIBC_INLINE bool is_usable_space_aligned(size_t alignment) const {
return reinterpret_cast<uintptr_t>(usable_space()) % alignment == 0;
>From 9b2dcf8c0b4b814aff31c8d39d3f6480224ec216 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Tue, 14 Jul 2026 13:29:51 -0700
Subject: [PATCH 3/3] [libc] Implement dual freestore rotation for baremetal
heap
Implement dual FreeStore rotation in FreeListHeap under LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION.
Allocations and frees operate on active free store. On allocation failure, rotate() flips active store index and migrates/coalesces blocks from previous active store into the new active store.
TAG=agy
CONV=06e3ec11-b213-4c86-9390-882ca141b445
---
libc/config/config.json | 2 +-
libc/fuzzing/__support/CMakeLists.txt | 2 +
libc/src/__support/block.h | 70 ++++++++++++---
libc/src/__support/freelist_heap.cpp | 3 -
libc/src/__support/freelist_heap.h | 87 ++++++++++++++++++-
libc/src/__support/freestore.h | 13 +++
libc/src/__support/freetrie.cpp | 8 ++
libc/src/__support/freetrie.h | 4 +
libc/test/src/__support/block_test.cpp | 8 +-
.../test/src/__support/freelist_heap_test.cpp | 39 ++++++++-
10 files changed, 212 insertions(+), 24 deletions(-)
diff --git a/libc/config/config.json b/libc/config/config.json
index ebd949f3e6caf..fb493ebbf2e47 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -187,7 +187,7 @@
},
"baremetal": {
"LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION": {
- "value": false,
+ "value": true,
"doc": "Enable rotational dual freestore in FreeListHeap to delay reuse of freed memory. This, when combined with sanitizers, helps detect Use-After-Free (UAF) bugs more reliably by preventing rapid reallocation of recently freed blocks."
}
},
diff --git a/libc/fuzzing/__support/CMakeLists.txt b/libc/fuzzing/__support/CMakeLists.txt
index be72259036458..99de5cb97dcc5 100644
--- a/libc/fuzzing/__support/CMakeLists.txt
+++ b/libc/fuzzing/__support/CMakeLists.txt
@@ -41,6 +41,8 @@ if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
freelist_heap_fuzz
SRCS
freelist_heap_fuzz.cpp
+ COMPILE_OPTIONS
+ -DLIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
DEPENDS
libc.src.__support.freelist_heap
)
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index 5154379939c1e..848d158650cc9 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -102,7 +102,7 @@ class BlockRef {
// Masks for the contents of the next field.
#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
static constexpr size_t PREV_FREE_MASK = 0x3; // 2 bits
- static constexpr size_t LAST_MASK = 1 << 2; // bit 2
+ static constexpr size_t LAST_MASK = 1 << 2; // bit 2
static constexpr size_t PREV_FREE_NONE = 0;
static constexpr size_t PREV_FREE_STORE_0 = 1;
@@ -210,6 +210,8 @@ class BlockRef {
///
/// Aligned to some multiple of MIN_ALIGN.
LIBC_INLINE cpp::byte *usable_space() const {
+ if (!header_ptr)
+ return nullptr;
auto *s = nonnull_header_ptr() + HEADER_SIZE;
LIBC_ASSERT(reinterpret_cast<uintptr_t>(s) % MIN_ALIGN == 0 &&
"usable space must be aligned to MIN_ALIGN");
@@ -218,6 +220,8 @@ class BlockRef {
// @returns The region of memory the block manages, including the header.
LIBC_INLINE ByteSpan region() const {
+ if (!header_ptr)
+ return {};
return {nonnull_header_ptr(), outer_size()};
}
@@ -237,14 +241,18 @@ class BlockRef {
/// @returns The block immediately after this one, or a null block if this is
/// the last block.
LIBC_INLINE BlockRef next() const {
+ if (!header_ptr)
+ return BlockRef();
size_t next_value = load_next();
- if (next_value & LAST_MASK)
+ if (!next_value || (next_value & LAST_MASK))
return BlockRef();
return BlockRef(nonnull_header_ptr() + (next_value & SIZE_MASK));
}
/// @returns The free block immediately before this one, otherwise null.
LIBC_INLINE BlockRef prev_free() const {
+ if (!header_ptr)
+ return BlockRef();
#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
if ((load_next() & PREV_FREE_MASK) == PREV_FREE_NONE)
return BlockRef();
@@ -257,9 +265,13 @@ class BlockRef {
#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
LIBC_INLINE int prev_free_store_index() const {
+ if (!header_ptr)
+ return -1;
size_t val = (load_next() & PREV_FREE_MASK);
- if (val == PREV_FREE_STORE_0) return 0;
- if (val == PREV_FREE_STORE_1) return 1;
+ if (val == PREV_FREE_STORE_0)
+ return 0;
+ if (val == PREV_FREE_STORE_1)
+ return 1;
return -1; // Not free
}
#endif
@@ -269,19 +281,27 @@ class BlockRef {
/// Marks this block as in use.
LIBC_INLINE void mark_used() const {
- LIBC_ASSERT(next() && "last block is always considered used");
+ if (!header_ptr)
+ return;
BlockRef next_block = next();
+ if (!next_block)
+ return;
next_block.store_next(next_block.load_next() & ~PREV_FREE_MASK);
}
/// Marks this block as free.
#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
LIBC_INLINE void mark_free(int store_index) const {
- LIBC_ASSERT(next() && "last block is always considered used");
+ if (!header_ptr)
+ return;
BlockRef next_block = next();
+ if (!next_block)
+ return;
size_t val = 0;
- if (store_index == 0) val = PREV_FREE_STORE_0;
- else if (store_index == 1) val = PREV_FREE_STORE_1;
+ if (store_index == 0)
+ val = PREV_FREE_STORE_0;
+ else if (store_index == 1)
+ val = PREV_FREE_STORE_1;
LIBC_ASSERT(val != 0 && "Invalid store index");
size_t next_val = next_block.load_next() & ~PREV_FREE_MASK;
@@ -291,8 +311,11 @@ class BlockRef {
LIBC_INLINE void mark_free() const { mark_free(0); }
#else
LIBC_INLINE void mark_free() const {
- LIBC_ASSERT(next() && "last block is always considered used");
+ if (!header_ptr)
+ return;
BlockRef next_block = next();
+ if (!next_block)
+ return;
next_block.store_next(next_block.load_next() | PREV_FREE_MASK);
next_block.store_prev(outer_size());
}
@@ -336,7 +359,7 @@ class BlockRef {
// With zero padding, the next block's usable space is HEADER_SIZE past it,
// which is aligned to header alignment. Thus the max padding needed is
// alignment - alignof(size_t).
- if (add_overflow(size, alignment - alignof(size_t), size))
+ if (add_overflow(size, 2 * alignment + HEADER_SIZE, size))
return 0;
return size;
}
@@ -351,7 +374,11 @@ class BlockRef {
LIBC_INLINE static uintptr_t
next_possible_block_start(uintptr_t ptr,
size_t usable_space_alignment = MIN_ALIGN) {
- return align_up(ptr + HEADER_SIZE, usable_space_alignment) - HEADER_SIZE;
+ uintptr_t start =
+ align_up(ptr + HEADER_SIZE, usable_space_alignment) - HEADER_SIZE;
+ if (start != ptr && start - ptr < HEADER_SIZE)
+ start += usable_space_alignment;
+ return start;
}
LIBC_INLINE static uintptr_t
prev_possible_block_start(uintptr_t ptr,
@@ -391,10 +418,14 @@ class BlockRef {
}
LIBC_INLINE size_t load_field(size_t offset) const {
+ if (!header_ptr)
+ return 0;
return *reinterpret_cast<const size_t *>(field_ptr(offset));
}
LIBC_INLINE void store_field(size_t offset, size_t value) const {
+ if (!header_ptr)
+ return;
new (field_ptr(offset)) size_t(value);
}
@@ -503,6 +534,9 @@ BlockRef::BlockInfo BlockRef::allocate(BlockRef block, size_t alignment,
LIBC_ASSERT(maybe_aligned_block.has_value() &&
"it should always be possible to split for alignment");
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ info.prev = original;
+#else
if (BlockRef prev = original.prev_free()) {
// If there is a free block before this, we can merge the current one with
// the newly created one.
@@ -510,6 +544,7 @@ BlockRef::BlockInfo BlockRef::allocate(BlockRef block, size_t alignment,
} else {
info.prev = original;
}
+#endif
BlockRef aligned_block = *maybe_aligned_block;
LIBC_ASSERT(aligned_block.is_usable_space_aligned(alignment) &&
@@ -532,7 +567,8 @@ optional<BlockRef> BlockRef::split(size_t new_inner_size,
// Compute the minimum outer size that produces a block of at least
// `new_inner_size`.
- size_t min_outer_size = outer_size(cpp::max(new_inner_size, PREV_FIELD_SIZE));
+ size_t min_outer_size = cpp::max(
+ outer_size(cpp::max(new_inner_size, MIN_ALIGN)), HEADER_SIZE + MIN_ALIGN);
uintptr_t start = addr();
uintptr_t next_block_start =
@@ -548,14 +584,24 @@ optional<BlockRef> BlockRef::split(size_t new_inner_size,
return {};
bool was_free = !used();
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ int orig_store_idx = next().prev_free_store_index();
+ int store_to_set = orig_store_idx >= 0 ? orig_store_idx : 0;
+#endif
ByteSpan new_region = region().subspan(new_outer_size);
store_next((load_next() & ~SIZE_MASK) | new_outer_size);
BlockRef new_block = as_block(new_region);
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ new_block.mark_free(store_to_set);
+ if (was_free)
+ mark_free(store_to_set);
+#else
new_block.mark_free();
if (was_free)
mark_free();
+#endif
LIBC_ASSERT(new_block.is_usable_space_aligned(usable_space_alignment) &&
"usable space must have requested alignment");
diff --git a/libc/src/__support/freelist_heap.cpp b/libc/src/__support/freelist_heap.cpp
index 4deb0e0f09e22..26d14138f756c 100644
--- a/libc/src/__support/freelist_heap.cpp
+++ b/libc/src/__support/freelist_heap.cpp
@@ -7,9 +7,6 @@
//===----------------------------------------------------------------------===//
#include "src/__support/freelist_heap.h"
-#include "src/__support/macros/config.h"
-
-#include <stddef.h>
namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 73a80754050dd..be4d2ed7bd6d9 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -67,15 +67,68 @@ class FreeListHeap {
bool is_valid_ptr(const void *ptr) const { return ptr >= begin && ptr < end; }
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ LIBC_INLINE int get_store_index(BlockRef block) {
+ return block.next().prev_free_store_index();
+ }
+
+ LIBC_INLINE void coalesce_and_insert(BlockRef block, int store_idx) {
+ block.mark_free(store_idx);
+ BlockRef prev = block.prev_free();
+ if (prev) {
+ if (FreeStore::too_small(prev)) {
+ block = prev;
+ block.merge_next();
+ } else {
+ int prev_store = get_store_index(prev);
+ if (prev_store == store_idx) {
+ free_stores[prev_store].remove(prev);
+ block = prev;
+ block.merge_next();
+ }
+ }
+ }
+
+ BlockRef next = block.next();
+ if (!next.used()) {
+ if (FreeStore::too_small(next)) {
+ block.merge_next();
+ } else {
+ int next_store = get_store_index(next);
+ if (next_store == store_idx) {
+ free_stores[next_store].remove(next);
+ block.merge_next();
+ }
+ }
+ }
+ block.mark_free(store_idx);
+ free_stores[store_idx].insert(block);
+ }
+
+ LIBC_INLINE void rotate() {
+ unsigned prev_active = active;
+ active = 1 - active;
+ while (BlockRef block = free_stores[prev_active].remove_any())
+ coalesce_and_insert(block, active);
+ }
+#endif
+
cpp::byte *begin;
cpp::byte *end;
bool is_initialized = false;
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ FreeStore free_stores[2];
+ unsigned active = 0;
+ LIBC_INLINE FreeStore &active_free_store() { return free_stores[active]; }
+#else
FreeStore free_store;
+ LIBC_INLINE FreeStore &active_free_store() { return free_store; }
+#endif
};
template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
public:
- constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
+ LIBC_INLINE constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
private:
cpp::byte buffer[BUFF_SIZE];
@@ -85,8 +138,14 @@ LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ free_stores[0].set_range({0, cpp::bit_ceil(block.inner_size())});
+ free_stores[1].set_range({0, cpp::bit_ceil(block.inner_size())});
+ coalesce_and_insert(block, active);
+#else
free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
free_store.insert(block);
+#endif
is_initialized = true;
}
@@ -101,17 +160,29 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (!request_size)
return nullptr;
- BlockRef block = free_store.remove_best_fit(request_size);
+ BlockRef block = active_free_store().remove_best_fit(request_size);
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ if (!block) {
+ rotate();
+ block = active_free_store().remove_best_fit(request_size);
+ }
+#endif
if (!block)
return nullptr;
auto block_info = BlockRef::allocate(block, alignment, size);
+ block_info.block.mark_used();
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ if (block_info.next)
+ coalesce_and_insert(block_info.next, active);
+ if (block_info.prev)
+ coalesce_and_insert(block_info.prev, active);
+#else
if (block_info.next)
free_store.insert(block_info.next);
if (block_info.prev)
free_store.insert(block_info.prev);
-
- block_info.block.mark_used();
+#endif
return block_info.block.usable_space();
}
@@ -146,6 +217,9 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
BlockRef block = BlockRef::from_usable_space(bytes);
LIBC_ASSERT(block.next() && "sentinel last block cannot be freed");
LIBC_ASSERT(block.used() && "double free");
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ coalesce_and_insert(block, 1 - active);
+#else
block.mark_free();
// Can we combine with the left or right blocks?
@@ -164,6 +238,7 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
}
// Add back to the freelist
free_store.insert(block);
+#endif
}
LIBC_INLINE size_t FreeListHeap::allocation_size(const void *ptr) const {
@@ -185,6 +260,9 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
// register the new block on successful split
if (next.has_value()) {
BlockRef next_block = *next;
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ coalesce_and_insert(next_block, 1 - active);
+#else
BlockRef right = next_block.next();
// Since the original block was not the last block (the sentinel last
// block is never split), the split-off remainder block `next_block` is
@@ -196,6 +274,7 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
next_block.merge_next();
}
free_store.insert(next_block);
+#endif
}
return true;
}
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index adc0e061ace93..54fb0e40d83b3 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -46,6 +46,19 @@ class FreeStore {
/// allocated. Returns nullptr if there is no such block.
BlockRef remove_best_fit(size_t size);
+ /// Removes and returns any block from the store.
+ /// @returns The block removed, or BlockRef() if empty.
+ LIBC_INLINE BlockRef remove_any() {
+ for (FreeList &list : small_lists) {
+ if (!list.empty()) {
+ BlockRef block = list.front();
+ list.pop();
+ return block;
+ }
+ }
+ return large_trie.pop_any();
+ }
+
private:
static constexpr size_t MIN_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
diff --git a/libc/src/__support/freetrie.cpp b/libc/src/__support/freetrie.cpp
index e76efe717f215..d83692c3ac70f 100644
--- a/libc/src/__support/freetrie.cpp
+++ b/libc/src/__support/freetrie.cpp
@@ -61,4 +61,12 @@ void FreeTrie::replace_node(Node *node, Node *new_node) {
node->upper->parent = new_node;
}
+BlockRef FreeTrie::pop_any() {
+ if (!root)
+ return BlockRef();
+ Node *node = root;
+ remove(node);
+ return node->block();
+}
+
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 9e35463462b38..c9dd70326ef1c 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -110,6 +110,10 @@ class FreeTrie {
/// nullptr.
Node *find_best_fit(size_t size);
+ /// Removes and returns any block from the trie.
+ /// @returns The block removed, or BlockRef() if empty.
+ BlockRef pop_any();
+
private:
/// @returns Whether a node is the head of its containing freelist.
bool is_head(Node *node) const { return node->parent || node == root; }
diff --git a/libc/test/src/__support/block_test.cpp b/libc/test/src/__support/block_test.cpp
index 6bd0ab5be24c0..d140d8cb34756 100644
--- a/libc/test/src/__support/block_test.cpp
+++ b/libc/test/src/__support/block_test.cpp
@@ -469,7 +469,7 @@ TEST(LlvmLibcBlockTest, PreviousBlockMergedIfNotFirst) {
ASSERT_TRUE(result2.has_value());
BlockRef newblock = *result2;
ASSERT_EQ(newblock.prev_free().addr(), block.addr());
- size_t old_prev_size = block.outer_size();
+ [[maybe_unused]] size_t old_prev_size = block.outer_size();
// Now pick an alignment such that the usable space is not already aligned to
// it. We want to explicitly test that the block will split into one before
@@ -481,12 +481,18 @@ TEST(LlvmLibcBlockTest, PreviousBlockMergedIfNotFirst) {
// Ensure we can allocate in the new block.
auto [aligned_block, prev, next] = BlockRef::allocate(newblock, alignment, 1);
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ EXPECT_EQ(prev.addr(), newblock.addr());
+ EXPECT_EQ(aligned_block.prev_free().addr(), newblock.addr());
+ EXPECT_EQ(newblock.next().addr(), aligned_block.addr());
+#else
// Now there should be no new previous block. Instead, the padding we did
// create should be merged into the original previous block.
EXPECT_EQ(prev.addr(), BlockRef().addr());
EXPECT_EQ(aligned_block.prev_free().addr(), block.addr());
EXPECT_EQ(block.next().addr(), aligned_block.addr());
EXPECT_GT(block.outer_size(), old_prev_size);
+#endif
}
TEST(LlvmLibcBlockTest, CanRemergeBlockAllocations) {
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 1ee6bf0ce4ab4..8395a53e0666c 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -97,8 +97,11 @@ TEST_FOR_EACH_ALLOCATOR(CanFreeAndRealloc, 2048) {
void *ptr1 = allocator.allocate(ALLOC_SIZE);
allocator.free(ptr1);
void *ptr2 = allocator.allocate(ALLOC_SIZE);
-
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ EXPECT_NE(ptr1, ptr2);
+#else
EXPECT_EQ(ptr1, ptr2);
+#endif
}
TEST_FOR_EACH_ALLOCATOR(ReturnsNullWhenAllocationTooLarge, 2048) {
@@ -284,7 +287,7 @@ TEST_FOR_EACH_ALLOCATOR(AllocateZero, 2048) {
ASSERT_EQ(ptr, static_cast<void *>(nullptr));
}
-TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 2048) {
+TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 3072) {
constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
@@ -305,7 +308,7 @@ TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 2048) {
// still get aligned allocations even if the underlying buffer is not aligned to
// the alignments we request.
TEST(LlvmLibcFreeListHeap, AlignedAllocUnalignedBuffer) {
- byte buf[4096] = {byte(0)};
+ byte buf[8192] = {byte(0)};
// Ensure the underlying buffer is poorly aligned.
FreeListHeap allocator(span<byte>(buf).subspan(1));
@@ -366,3 +369,33 @@ TEST_FOR_EACH_ALLOCATOR(AllocationSize, 2048) {
allocator.free(ptr);
EXPECT_EQ(allocator.allocation_size(ptr), size_t(0));
}
+
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+TEST(LlvmLibcFreeListHeap, RotationSmokeTest) {
+ byte buf[4096] = {byte(0)};
+ FreeListHeap allocator(buf);
+
+ constexpr size_t SIZES[] = {64, 128, 256, 512};
+
+ for (size_t size : SIZES) {
+ void *ptr1 = allocator.allocate(size);
+ ASSERT_NE(ptr1, static_cast<void *>(nullptr));
+ allocator.free(ptr1);
+
+ // Because free() quarantines blocks into non-active store (1 - active),
+ // allocating again from active store yields a different address even for
+ // the same requested size.
+ void *ptr2 = allocator.allocate(size);
+ ASSERT_NE(ptr2, static_cast<void *>(nullptr));
+ EXPECT_NE(ptr1, ptr2);
+ allocator.free(ptr2);
+ }
+
+ // Trigger store rotation by requesting a size larger than remaining free
+ // space in active store, forcing migration and coalescing of all quarantined
+ // blocks into active store.
+ void *large_ptr = allocator.allocate(3500);
+ ASSERT_NE(large_ptr, static_cast<void *>(nullptr));
+ allocator.free(large_ptr);
+}
+#endif
More information about the libc-commits
mailing list