[libc-commits] [libc] b47a631 - [libc] Implement dual freestore rotation for baremetal heap (#209811)
via libc-commits
libc-commits at lists.llvm.org
Tue Sep 22 18:55:18 PDT 2026
Author: Schrodinger ZHU Yifan
Date: 2026-09-22T18:55:11-07:00
New Revision: b47a631eeaff515d63232ba5b69f7206ea9ca718
URL: https://github.com/llvm/llvm-project/commit/b47a631eeaff515d63232ba5b69f7206ea9ca718
DIFF: https://github.com/llvm/llvm-project/commit/b47a631eeaff515d63232ba5b69f7206ea9ca718.diff
LOG: [libc] Implement dual freestore rotation for baremetal heap (#209811)
Implement dual FreeStore rotation in FreeListHeap under
LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION option for baremetal
targets.
- Allocations pull from active store; free() quarantines blocks into
non-active store (1 - active).
- On allocation failure in active store, rotate() flips active index and
migrates/coalesces quarantined blocks into the new active store.
- Added 2-bit prev_free tracking in BlockRef metadata to distinguish
freestore indices.
- Added unit smoke tests and updated fuzzer for dual freestore rotation.
Assisted-by: Gemini and Claude based automation tool (human-in-the-loop)
---------
Co-authored-by: Yifan Zhu <yfzhu at google.com>
Co-authored-by: Claude Fable 5.1 <noreply at anthropic.com>
Added:
Modified:
libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
libc/config/config.json
libc/fuzzing/__support/CMakeLists.txt
libc/src/__support/CMakeLists.txt
libc/src/__support/block.h
libc/src/__support/freelist_heap.h
libc/src/__support/freestore.h
libc/src/__support/freetrie.cpp
libc/src/__support/freetrie.h
libc/src/__support/tlsf_table.h
libc/test/src/__support/CMakeLists.txt
libc/test/src/__support/freelist_heap_test.cpp
Removed:
################################################################################
diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index 99defb24d249ae..79f2d80bcba7a1 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -189,6 +189,10 @@ function(_get_compile_options_from_config output_var)
libc_add_definition(config_options "LIBC_COPT_PRINTF_DISABLE_BITINT")
endif()
+ if(LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION)
+ libc_add_definition(config_options "LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION")
+ endif()
+
if(LIBC_COPT_USE_C_ASSERT)
list(APPEND config_options "-DLIBC_COPT_USE_C_ASSERT")
endif()
diff --git a/libc/config/config.json b/libc/config/config.json
index fd7784d3d3e556..604b03c9b5688a 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -189,6 +189,12 @@
"doc": "Trap with SIGFPE when feraiseexcept is called with unmasked floating point exceptions, similar to glibc's behavior. This is currently working only on x86 with SSE."
}
},
+ "baremetal": {
+ "LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION": {
+ "value": false,
+ "doc": "Enable rotational dual freestore in FreeListHeap to delay reuse of freed memory. This, when combined with sanitizers, helps detect Use-After-Free (UAF) bugs more reliably by preventing rapid reallocation of recently freed blocks. It costs one extra flag bit in the block header, which doubles the minimum allocation alignment on 32-bit targets."
+ }
+ },
"assert": {
"LIBC_COPT_USE_C_ASSERT": {
"value": false,
diff --git a/libc/fuzzing/__support/CMakeLists.txt b/libc/fuzzing/__support/CMakeLists.txt
index be722590364580..2fbaa2f7cd6cff 100644
--- a/libc/fuzzing/__support/CMakeLists.txt
+++ b/libc/fuzzing/__support/CMakeLists.txt
@@ -44,11 +44,25 @@ if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
DEPENDS
libc.src.__support.freelist_heap
)
+ # Same fuzzer, but with the heap rotating between several free stores, so
+ # that both configurations get covered by a single build.
+ add_libc_fuzzer(
+ freelist_heap_rotation_fuzz
+ SRCS
+ freelist_heap_fuzz.cpp
+ COMPILE_OPTIONS
+ -DLIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ DEPENDS
+ libc.src.__support.freelist_heap
+ )
# TODO(#119995): Remove this once sccache on Windows no longer requires
# the use of -DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded.
get_fq_target_name(freelist_heap_fuzz freelist_heap_fuzz_target_name)
+ get_fq_target_name(freelist_heap_rotation_fuzz
+ freelist_heap_rotation_fuzz_target_name)
set_target_properties(
${freelist_heap_fuzz_target_name}
+ ${freelist_heap_rotation_fuzz_target_name}
PROPERTIES
MSVC_DEBUG_INFORMATION_FORMAT ""
)
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index f6d998fee2ef66..18fdf72f19fa24 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -17,6 +17,7 @@ add_header_library(
DEPENDS
libc.hdr.stdint_proxy
libc.src.__support.CPP.algorithm
+ libc.src.__support.CPP.bit
libc.src.__support.CPP.limits
libc.src.__support.CPP.new
libc.src.__support.CPP.optional
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index be2a71f32a23f1..d002dae2c16218 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -17,6 +17,7 @@
#include "hdr/stdint_proxy.h"
#include "hdr/types/size_t.h"
#include "src/__support/CPP/algorithm.h"
+#include "src/__support/CPP/bit.h"
#include "src/__support/CPP/cstddef.h"
#include "src/__support/CPP/limits.h"
#include "src/__support/CPP/new.h"
@@ -99,9 +100,22 @@ using cpp::optional;
/// The next offset of a block matches the previous offset of its next block.
/// The first block in a list is denoted by having a previous offset of `0`.
class BlockRef {
- // Masks for the contents of the next field.
- static constexpr size_t PREV_FREE_MASK = 1 << 0;
- static constexpr size_t LAST_MASK = 1 << 1;
+public:
+ /// The number of free stores that free blocks are distributed over.
+ ///
+ /// Every free block records which store owns it, which lets an allocator
+ /// hold several stores and rotate between them; see FreeListHeap. Each
+ /// additional store widens the free field below, and hence MIN_ALIGN.
+#ifdef LIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ static constexpr size_t NUM_FREE_STORES = 2;
+#else
+ static constexpr size_t NUM_FREE_STORES = 1;
+#endif
+
+private:
+ static constexpr size_t PREV_FREE_BITS = cpp::bit_width(NUM_FREE_STORES);
+ static constexpr size_t PREV_FREE_MASK = (size_t{1} << PREV_FREE_BITS) - 1;
+ static constexpr size_t LAST_MASK = size_t{1} << PREV_FREE_BITS;
static constexpr size_t SIZE_MASK = ~(PREV_FREE_MASK | LAST_MASK);
// Header field offsets. The value at PREV_OFFSET is only meaningful when the
@@ -112,10 +126,10 @@ class BlockRef {
public:
static constexpr size_t HEADER_SIZE = NEXT_OFFSET + sizeof(size_t);
- // To ensure block sizes have two lower unused bits, ensure usable space is
- // always aligned to at least 4 bytes. (The distances between usable spaces,
- // the outer size, is then always also 4-aligned.)
- static constexpr size_t MIN_ALIGN = cpp::max(size_t{4}, alignof(max_align_t));
+ // The flag bits above live in the lower bits of a block's size, so the
+ // usable space must be aligned to the first bit they leave unused.
+ static constexpr size_t MIN_ALIGN =
+ cpp::max(LAST_MASK << 1, alignof(max_align_t));
LIBC_INLINE constexpr BlockRef() = default;
LIBC_INLINE explicit constexpr BlockRef(cpp::byte *header_ptr)
@@ -237,6 +251,12 @@ class BlockRef {
return BlockRef(nonnull_header_ptr() - load_prev());
}
+ /// @returns The index of the free store owning the block immediately before
+ /// this one, or a negative value if that block is not free.
+ LIBC_INLINE int prev_free_store_index() const {
+ return static_cast<int>(load_next() & PREV_FREE_MASK) - 1;
+ }
+
/// @returns Whether the block is unavailable for allocation.
LIBC_INLINE bool used() const { return !next() || !next().prev_free(); }
@@ -247,11 +267,14 @@ class BlockRef {
next_block.store_next(next_block.load_next() & ~PREV_FREE_MASK);
}
- /// Marks this block as free.
- LIBC_INLINE void mark_free() const {
+ /// Marks this block as free and owned by the given free store.
+ LIBC_INLINE void mark_free(size_t store_index = 0) const {
LIBC_ASSERT(next() && "last block is always considered used");
+ LIBC_ASSERT(store_index < NUM_FREE_STORES && "invalid free store index");
BlockRef next_block = next();
- next_block.store_next(next_block.load_next() | PREV_FREE_MASK);
+ // Replace the free field with `store_index + 1`, as 0 encodes "in use".
+ next_block.store_next((next_block.load_next() & ~PREV_FREE_MASK) |
+ (store_index + 1));
next_block.store_prev(outer_size());
}
@@ -460,9 +483,13 @@ BlockRef::BlockInfo BlockRef::allocate(BlockRef block, size_t alignment,
LIBC_ASSERT(maybe_aligned_block.has_value() &&
"it should always be possible to split for alignment");
- if (BlockRef prev = original.prev_free()) {
- // If there is a free block before this, we can merge the current one with
- // the newly created one.
+ // If the block before the padding is free and owned by the same free store,
+ // the two can be merged; blocks owned by
diff erent stores must be kept
+ // apart. Otherwise the padding is handed back to the caller, which decides
+ // which store it belongs to.
+ BlockRef prev = original.prev_free();
+ if (prev && original.prev_free_store_index() ==
+ original.next().prev_free_store_index()) {
prev.merge_next();
} else {
info.prev = original;
@@ -505,14 +532,20 @@ optional<BlockRef> BlockRef::split(size_t new_inner_size,
return {};
bool was_free = !used();
+ // The block split off below is always free. If this block is free too, both
+ // halves must stay in the same store to remain mergeable; if it is in use,
+ // the caller owns the new block and will move it to whichever store it
+ // wants, so any valid index will do.
+ size_t store_index =
+ was_free ? static_cast<size_t>(next().prev_free_store_index()) : 0;
ByteSpan new_region = region().subspan(new_outer_size);
store_next((load_next() & ~SIZE_MASK) | new_outer_size);
BlockRef new_block = as_block(new_region);
- new_block.mark_free();
+ new_block.mark_free(store_index);
if (was_free)
- mark_free();
+ mark_free(store_index);
LIBC_ASSERT(new_block.is_usable_space_aligned(usable_space_alignment) &&
"usable space must have requested alignment");
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index d2ec9339d72ed2..74c13332486c6f 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -38,6 +38,21 @@ LIBC_INLINE constexpr bool IsPow2(size_t x) { return x && (x & (x - 1)) == 0; }
class FreeListHeap {
public:
+ /// The heap keeps its free blocks in NUM_FREE_STORES stores and rotates
+ /// between them: allocations are served from the active store, while frees
+ /// go to the next one, quarantining the memory there. Only once the active
+ /// store cannot satisfy a request does rotate() make the quarantined memory
+ /// available again. Delaying reuse this way makes use-after-free bugs much
+ /// more likely to be caught, e.g. by a sanitizer.
+ ///
+ /// A request the active store cannot serve thus ends the quarantine period
+ /// early, whether or not it succeeds afterwards.
+ ///
+ /// With a single store (the default) the quarantine store is the active
+ /// store, no rotation ever happens, and this degenerates into the usual
+ /// immediate-reuse behavior.
+ static constexpr size_t NUM_FREE_STORES = BlockRef::NUM_FREE_STORES;
+
constexpr FreeListHeap() : begin(&_end), end(&__llvm_libc_heap_limit) {}
constexpr FreeListHeap(span<cpp::byte> region)
@@ -51,7 +66,10 @@ class FreeListHeap {
void *realloc(void *ptr, size_t size);
void *calloc(size_t num, size_t size);
size_t allocation_size(const void *ptr) const;
- LIBC_INLINE void integrity_check() const { free_store.integrity_check(); }
+ LIBC_INLINE void integrity_check() const {
+ for (const FreeStore &store : free_stores)
+ store.integrity_check();
+ }
cpp::span<cpp::byte> region() const { return {begin, end}; }
@@ -68,15 +86,80 @@ class FreeListHeap {
bool is_valid_ptr(const void *ptr) const { return ptr >= begin && ptr < end; }
+ /// The store that allocations are served from.
+ LIBC_INLINE FreeStore &active_free_store() { return free_stores[active]; }
+
+ /// @returns The index of the store that receives newly freed blocks. With a
+ /// single free store this is the active store itself, so freed memory is
+ /// immediately available again.
+ LIBC_INLINE size_t quarantine_store_index() const {
+ return (active + 1) % NUM_FREE_STORES;
+ }
+
+ /// Whether the free block `neighbor` can be merged into a block owned by
+ /// `store_index`. Merging across stores would hand quarantined memory back
+ /// out through the active store, so only blocks of the same store merge,
+ /// plus blocks too small for any store to track (see FreeStore::too_small),
+ /// which are owned by no store.
+ LIBC_INLINE static bool can_merge(BlockRef neighbor, size_t store_index) {
+ return FreeStore::too_small(neighbor) ||
+ neighbor.next().prev_free_store_index() ==
+ static_cast<int>(store_index);
+ }
+
+ /// Inserts `block` into store `store_index`, coalescing it with the
+ /// neighbors that can be merged into that store. The block may be in use or
+ /// free in another store.
+ LIBC_INLINE void insert(BlockRef block, size_t store_index) {
+ block.mark_free(store_index);
+
+ // Absorbing an untracked neighbor may expose another mergeable one beyond
+ // it, so keep going.
+ for (BlockRef prev = block.prev_free();
+ prev && can_merge(prev, store_index); prev = block.prev_free()) {
+ free_stores[store_index].remove(prev);
+ block = prev;
+ block.merge_next();
+ }
+
+ for (BlockRef next = block.next();
+ !next.used() && can_merge(next, store_index); next = block.next()) {
+ free_stores[store_index].remove(next);
+ block.merge_next();
+ }
+
+ // Merging moved the block boundaries, and an absorbed block may have been
+ // owned by another store, so record the ownership of the result again.
+ block.mark_free(store_index);
+ free_stores[store_index].insert(block);
+ }
+
+ /// Ends the current quarantine period: the store that has been collecting
+ /// freed blocks becomes the active one, and whatever is left in the
+ /// previously active store is migrated (and coalesced) into it, so that the
+ /// newly active store holds all the free memory of the heap. Returns false
+ /// without doing anything if nothing is quarantined.
+ LIBC_INLINE bool rotate() {
+ if (NUM_FREE_STORES < 2 || free_stores[quarantine_store_index()].empty())
+ return false;
+
+ size_t prev_active = active;
+ active = quarantine_store_index();
+ while (BlockRef block = free_stores[prev_active].remove_any())
+ insert(block, active);
+ return true;
+ }
+
cpp::byte *begin;
cpp::byte *end;
bool is_initialized = false;
- FreeStore free_store;
+ FreeStore free_stores[NUM_FREE_STORES];
+ size_t active = 0;
};
template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
public:
- constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
+ LIBC_INLINE constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
private:
cpp::byte buffer[BUFF_SIZE];
@@ -86,8 +169,9 @@ LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
- free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
- free_store.insert(block);
+ for (FreeStore &store : free_stores)
+ store.set_range({0, cpp::bit_ceil(block.inner_size())});
+ free_stores[active].insert(block);
is_initialized = true;
}
@@ -99,20 +183,26 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
init();
size_t request_size = BlockRef::min_size_for_allocation(alignment, size);
- if (!request_size)
+ // Don't disturb the quarantine for a request the heap could never serve.
+ if (!request_size || request_size > region().size())
return nullptr;
- BlockRef block = free_store.remove_best_fit(request_size);
+ BlockRef block = active_free_store().remove_best_fit(request_size);
+ // The active store is out of memory; make the quarantined blocks available
+ // again and retry.
+ if (!block && rotate())
+ block = active_free_store().remove_best_fit(request_size);
if (!block)
return nullptr;
auto block_info = BlockRef::allocate(block, alignment, size);
+ block_info.block.mark_used();
+ // The leftovers of the block were never handed out, so they stay in the
+ // active store rather than being quarantined.
if (block_info.next)
- free_store.insert(block_info.next);
+ insert(block_info.next, active);
if (block_info.prev)
- free_store.insert(block_info.prev);
-
- block_info.block.mark_used();
+ insert(block_info.prev, active);
return block_info.block.usable_space();
}
@@ -147,24 +237,7 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
BlockRef block = BlockRef::from_usable_space(bytes);
LIBC_ASSERT(block.next() && "sentinel last block cannot be freed");
LIBC_ASSERT(block.used() && "double free");
- block.mark_free();
-
- // Can we combine with the left or right blocks?
- BlockRef prev_free = block.prev_free();
- BlockRef next = block.next();
-
- if (prev_free) {
- // Remove from free store and merge.
- free_store.remove(prev_free);
- block = prev_free;
- block.merge_next();
- }
- if (!next.used()) {
- free_store.remove(next);
- block.merge_next();
- }
- // Add back to the freelist
- free_store.insert(block);
+ insert(block, quarantine_store_index());
}
LIBC_INLINE size_t FreeListHeap::allocation_size(const void *ptr) const {
@@ -186,17 +259,13 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
// register the new block on successful split
if (next.has_value()) {
BlockRef next_block = *next;
- BlockRef right = next_block.next();
// Since the original block was not the last block (the sentinel last
// block is never split), the split-off remainder block `next_block` is
- // also not the last block. Thus, its next block `right` is guaranteed
- // to be non-null.
- LIBC_ASSERT(right && "right block must be non-null");
- if (!right.used()) {
- free_store.remove(right);
- next_block.merge_next();
- }
- free_store.insert(next_block);
+ // also not the last block. Thus, its next block is guaranteed to be
+ // non-null.
+ LIBC_ASSERT(next_block.next() && "right block must be non-null");
+ // The remainder is memory the caller has given up, so quarantine it.
+ insert(next_block, quarantine_store_index());
}
return true;
}
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index ccc4f9555c64a0..59e0de2380c13a 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -73,6 +73,44 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
list.integrity_check();
}
+ /// Removes and returns any block from the store.
+ /// @returns The block removed, or BlockRef() if empty.
+ LIBC_INLINE BlockRef remove_any() {
+ for (size_t i = 0; i < TOTAL_BITS - 1; ++i) {
+ if (!free_lists[i].empty()) {
+ BlockRef block = free_lists[i].front();
+ free_lists[i].pop();
+ if (free_lists[i].empty())
+ free_sizes.mark_vacant(i);
+ return block;
+ }
+ }
+ if constexpr (USE_TRIE) {
+ if (BlockRef block = trie.pop_any()) {
+ if (trie.empty())
+ free_sizes.mark_vacant(TOTAL_BITS - 1);
+ return block;
+ }
+ } else {
+ if (!overflow_list.empty()) {
+ BlockRef block = overflow_list.front();
+ overflow_list.pop();
+ if (overflow_list.empty())
+ free_sizes.mark_vacant(TOTAL_BITS - 1);
+ return block;
+ }
+ }
+ return BlockRef();
+ }
+
+ /// Whether `block` is too small to hold a free list node. insert() and
+ /// remove() ignore such blocks, so they are owned by no store.
+ LIBC_INLINE static bool too_small(BlockRef block) {
+ return block.outer_size() < MIN_OUTER_SIZE;
+ }
+
+ LIBC_INLINE bool empty() const { return free_sizes.empty(); }
+
private:
LIBC_INLINE constexpr TLSFFreeStoreImpl(cpp::bool_constant<true>) : trie() {}
LIBC_INLINE constexpr TLSFFreeStoreImpl(cpp::bool_constant<false>)
@@ -100,10 +138,6 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
}
protected:
- LIBC_INLINE static bool too_small(BlockRef block) {
- return block.outer_size() < MIN_OUTER_SIZE;
- }
-
Table free_sizes;
cpp::array<FreeList, TOTAL_BITS - 1> free_lists;
union {
diff --git a/libc/src/__support/freetrie.cpp b/libc/src/__support/freetrie.cpp
index d0392342f22316..f15664db2f12fd 100644
--- a/libc/src/__support/freetrie.cpp
+++ b/libc/src/__support/freetrie.cpp
@@ -79,4 +79,12 @@ void FreeTrie::integrity_check() const {
integrity_check_trie_node(integrity_check_trie_node, root());
}
+BlockRef FreeTrie::pop_any() {
+ if (!root_)
+ return BlockRef();
+ Node *node = root_;
+ remove(node);
+ return node->block();
+}
+
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 1be34f456021b0..bdb115eeb184b8 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -132,6 +132,10 @@ class FreeTrie {
/// Verify integrity of all nodes in the trie.
void integrity_check() const;
+ /// Removes and returns any block from the trie.
+ /// @returns The block removed, or BlockRef() if empty.
+ BlockRef pop_any();
+
private:
/// @returns Whether a node is the head of its containing freelist.
bool is_head(Node *node) const { return node->parent || node == root_; }
diff --git a/libc/src/__support/tlsf_table.h b/libc/src/__support/tlsf_table.h
index e75f1074ca4b32..906b75a32cd096 100644
--- a/libc/src/__support/tlsf_table.h
+++ b/libc/src/__support/tlsf_table.h
@@ -148,6 +148,13 @@ template <typename CONFIG> class TLSFTable {
lookup_table[entry_index] &= ~(uintptr_t(1) << bit_offset);
}
+ LIBC_INLINE bool empty() const {
+ for (uintptr_t entry : lookup_table)
+ if (entry != 0)
+ return false;
+ return true;
+ }
+
LIBC_INLINE bool is_occupied(size_t bin) const {
size_t entry_index = bin / BITS_PER_ENTRY;
size_t bit_offset = bin % BITS_PER_ENTRY;
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 3663b1774e2117..20be9fb492aa9b 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -82,6 +82,24 @@ if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
libc.src.string.memory_utils.inline_memset
)
+ # Same tests, but with the heap rotating between several free stores, so that
+ # both configurations get covered by a single build.
+ add_libc_test(
+ freelist_heap_rotation_test
+ SUITE
+ libc-support-tests
+ SRCS
+ freelist_heap_test.cpp
+ COMPILE_OPTIONS
+ -DLIBC_COPT_BAREMETAL_HEAP_ENABLE_FREESTORE_ROTATION
+ DEPENDS
+ libc.src.__support.CPP.span
+ libc.src.__support.freelist_heap
+ libc.src.string.memcmp
+ libc.src.string.memcpy
+ libc.src.string.memory_utils.inline_memset
+ )
+
add_libc_test(
freelist_heap_death_test
SUITE
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 80b81311744590..d7a1b406f69757 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -32,6 +32,7 @@ using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::freelist_heap;
using LIBC_NAMESPACE::FreeListHeap;
using LIBC_NAMESPACE::FreeListHeapBuffer;
+using LIBC_NAMESPACE::FreeStore;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::span;
@@ -97,8 +98,12 @@ TEST_FOR_EACH_ALLOCATOR(CanFreeAndRealloc, 2048) {
void *ptr1 = allocator.allocate(ALLOC_SIZE);
allocator.free(ptr1);
void *ptr2 = allocator.allocate(ALLOC_SIZE);
-
- EXPECT_EQ(ptr1, ptr2);
+ // The freed block is quarantined in the inactive free store, so it cannot
+ // be handed back out until the stores rotate.
+ if constexpr (FreeListHeap::NUM_FREE_STORES > 1)
+ EXPECT_NE(ptr1, ptr2);
+ else
+ EXPECT_EQ(ptr1, ptr2);
}
TEST_FOR_EACH_ALLOCATOR(ReturnsNullWhenAllocationTooLarge, 2048) {
@@ -284,7 +289,7 @@ TEST_FOR_EACH_ALLOCATOR(AllocateZero, 2048) {
ASSERT_EQ(ptr, static_cast<void *>(nullptr));
}
-TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 2048) {
+TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 3072) {
constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
@@ -305,7 +310,7 @@ TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 2048) {
// still get aligned allocations even if the underlying buffer is not aligned to
// the alignments we request.
TEST(LlvmLibcFreeListHeap, AlignedAllocUnalignedBuffer) {
- byte buf[4096] = {byte(0)};
+ byte buf[8192] = {byte(0)};
// Ensure the underlying buffer is poorly aligned.
FreeListHeap allocator(span<byte>(buf).subspan(1));
@@ -391,3 +396,111 @@ TEST_FOR_EACH_ALLOCATOR(IntegrityCheck, 2048) {
allocator.free(ptr2);
allocator.integrity_check();
}
+
+TEST(LlvmLibcFreeListHeap, RotationSmokeTest) {
+ if constexpr (FreeListHeap::NUM_FREE_STORES > 1) {
+ byte buf[4096] = {byte(0)};
+ FreeListHeap allocator(buf);
+
+ constexpr size_t SIZES[] = {64, 128, 256, 512};
+
+ for (size_t size : SIZES) {
+ void *ptr1 = allocator.allocate(size);
+ ASSERT_NE(ptr1, static_cast<void *>(nullptr));
+ allocator.free(ptr1);
+
+ // free() quarantines the block in the inactive store, so allocating the
+ // same size again yields a
diff erent address.
+ void *ptr2 = allocator.allocate(size);
+ ASSERT_NE(ptr2, static_cast<void *>(nullptr));
+ EXPECT_NE(ptr1, ptr2);
+ allocator.free(ptr2);
+ }
+
+ // Ask for more than the active store has left, which forces a rotation:
+ // all the quarantined blocks are migrated and coalesced into the store
+ // that is about to become active.
+ void *large_ptr = allocator.allocate(3500);
+ ASSERT_NE(large_ptr, static_cast<void *>(nullptr));
+ allocator.integrity_check();
+ allocator.free(large_ptr);
+ }
+}
+
+// Padding split off by an aligned allocation may be too small to track and
+// hence owned by no store. If left between a quarantined block A and the
+// aligned block B, freeing B into A's store must coalesce all three rather
+// than stop at [A][padding + B].
+TEST(LlvmLibcFreeListHeap, CoalescesAcrossUntrackedPadding) {
+ constexpr size_t MIN_ALIGN = BlockRef::MIN_ALIGN;
+ if constexpr (FreeListHeap::NUM_FREE_STORES > 1 &&
+ MIN_ALIGN < FreeStore::MIN_OUTER_SIZE) {
+ constexpr size_t N = 2048;
+ constexpr size_t ALIGNMENT = 2 * MIN_ALIGN;
+
+ // X's usable space must be one MIN_ALIGN unit short of ALIGNMENT; which
+ // size of A gets there depends on where the heap starts, so try a few.
+ bool tested = false;
+ for (size_t a_size = 2 * MIN_ALIGN; a_size <= 8 * MIN_ALIGN && !tested;
+ a_size += MIN_ALIGN) {
+ byte buf[N] = {byte(0)};
+ FreeListHeap allocator(buf);
+
+ void *a = allocator.allocate(a_size);
+ void *x = allocator.allocate(8 * MIN_ALIGN);
+ void *fence = allocator.allocate(MIN_ALIGN);
+ ASSERT_NE(a, static_cast<void *>(nullptr));
+ ASSERT_NE(x, static_cast<void *>(nullptr));
+ ASSERT_NE(fence, static_cast<void *>(nullptr));
+ if (reinterpret_cast<uintptr_t>(x) % ALIGNMENT != MIN_ALIGN)
+ continue;
+ tested = true;
+
+ // Quarantine X, rotate (a request for more than is free fails, but
+ // the stores rotate first), then quarantine A in the other store.
+ allocator.free(x);
+ ASSERT_EQ(allocator.allocate(N - 64), static_cast<void *>(nullptr));
+ allocator.free(a);
+
+ // Carve B out of X, leaving one MIN_ALIGN unit of untracked padding.
+ void *b = allocator.aligned_allocate(ALIGNMENT, ALIGNMENT);
+ ASSERT_NE(b, static_cast<void *>(nullptr));
+ ASSERT_EQ(b, static_cast<void *>(static_cast<byte *>(x) + MIN_ALIGN));
+
+ BlockRef block_a = BlockRef::from_usable_space(a);
+ BlockRef block_b = BlockRef::from_usable_space(b);
+ BlockRef padding = block_b.prev_free();
+ ASSERT_NE(padding.addr(), BlockRef().addr());
+ ASSERT_EQ(padding.prev_free().addr(), block_a.addr());
+ ASSERT_TRUE(FreeStore::too_small(padding));
+ ASSERT_FALSE(FreeStore::too_small(block_a));
+ BlockRef after_b = block_b.next();
+
+ allocator.free(b);
+ EXPECT_EQ(block_a.next().addr(), after_b.addr());
+ allocator.integrity_check();
+ }
+ EXPECT_TRUE(tested);
+ }
+}
+
+// A request larger than the whole heap can never be served, so it must fail
+// without ending the quarantine period.
+TEST(LlvmLibcFreeListHeap, OversizedRequestDoesNotRotate) {
+ if constexpr (FreeListHeap::NUM_FREE_STORES > 1) {
+ byte buf[2048] = {byte(0)};
+ FreeListHeap allocator(buf);
+
+ void *ptr1 = allocator.allocate(64);
+ ASSERT_NE(ptr1, static_cast<void *>(nullptr));
+ allocator.free(ptr1);
+
+ ASSERT_EQ(allocator.allocate(allocator.region().size() + 1),
+ static_cast<void *>(nullptr));
+
+ // The freed block is still quarantined, so it cannot be handed back out.
+ void *ptr2 = allocator.allocate(64);
+ ASSERT_NE(ptr2, static_cast<void *>(nullptr));
+ EXPECT_NE(ptr1, ptr2);
+ }
+}
More information about the libc-commits
mailing list