[libc-commits] [libc] [libc] Implement TLSF FreeStore with Trie Overflow Bin (PR #203415)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Fri Jul 31 10:23:25 PDT 2026
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/203415
>From 0c0c9f956e68e981c917822109bdfe83e761f7b5 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:19:19 -0700
Subject: [PATCH 01/12] [libc] Implement TLSF FreeStore
- Replace freestore.h with TLSF implementation.
- Add USE_TRIE_FOR_OVERFLOW_BIN option to use FreeTrie in TLSF.
- Add SizeRange+root constructor and get_root getter to FreeTrie.
- Remove obsolete set_range interface from FreeStore, FreeListHeap, and tests.
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freelist.h | 3 +
libc/src/__support/freelist_heap.h | 1 -
libc/src/__support/freestore.h | 376 +++++++++++++++++----
libc/src/__support/freetrie.h | 6 +-
libc/test/src/__support/freestore_test.cpp | 27 +-
libc/test/src/__support/freetrie_test.cpp | 16 +
6 files changed, 345 insertions(+), 84 deletions(-)
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index 48e70c7c29df6..ae2de684b3a24 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -41,6 +41,9 @@ class FreeList {
/// @returns The inner size of blocks in the list containing this node.
LIBC_INLINE size_t size() const { return block().inner_size(); }
+ /// @returns The next node in the list containing this node.
+ LIBC_INLINE Node *next_node() const { return next; }
+
private:
// Circularly linked pointers to adjacent nodes.
Node *prev;
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 73a80754050dd..d8d937eba50d8 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -85,7 +85,6 @@ LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
- free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
free_store.insert(block);
is_initialized = true;
}
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index adc0e061ace93..9394972f3adcc 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -7,113 +7,349 @@
//===----------------------------------------------------------------------===//
///
/// \file
-/// Interface for freestore.
+/// This file contains a two-level segregated fit free block store.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
#define LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
-#include "freetrie.h"
+#include "hdr/stdint_proxy.h"
+#include "hdr/types/size_t.h"
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/block.h"
+#include "src/__support/freelist.h"
+#include "src/__support/freetrie.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
namespace LIBC_NAMESPACE_DECL {
-/// A best-fit store of variously-sized free blocks. Blocks can be inserted and
-/// removed in logarithmic time.
-class FreeStore {
- friend class FreeListHeap;
-
-public:
- FreeStore() = default;
- FreeStore(const FreeStore &other) = delete;
- FreeStore &operator=(const FreeStore &other) = delete;
-
- /// Sets the range of possible block sizes. This can only be called when the
- /// trie is empty.
- LIBC_INLINE void set_range(FreeTrie::SizeRange range) {
- large_trie.set_range(range);
- }
-
- /// Insert a free block. If the block is too small to be tracked, nothing
- /// happens.
- void insert(BlockRef block);
+/// Configuration for TLSFFreeStore.
+template <size_t UNIT_SIZE_VAL, size_t STEP_SIZE_BITS_VAL,
+ size_t NUM_STEP_BITS_VAL, size_t NUM_TABLE_ENTRIES_VAL,
+ bool USE_TRIE_FOR_OVERFLOW_BIN_VAL = false>
+struct TLSFFreeStoreConfig {
+ static constexpr size_t UNIT_SIZE = UNIT_SIZE_VAL;
+ static constexpr size_t STEP_SIZE_BITS = STEP_SIZE_BITS_VAL;
+ static constexpr size_t NUM_STEP_BITS = NUM_STEP_BITS_VAL;
+ static constexpr size_t NUM_TABLE_ENTRIES = NUM_TABLE_ENTRIES_VAL;
+ static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN =
+ USE_TRIE_FOR_OVERFLOW_BIN_VAL;
+};
- /// Remove a free block. If the block is too small to be tracked, nothing
- /// happens.
- void remove(BlockRef block);
+// A two-level segregated fit store for free blocks.
+//
+// The store starts with small lists that grow linearly for small sizes, which
+// covers [0, ... UNIT_SIZE * EXP_BASE]. For larger sizes, the bits are managed
+// in a 2-D table. One can think of each row containing NUM_STEPS lists. Along
+// the row, the size grows by 2 exponentially; along the column, the size
+// increases by STEP_SIZE linearly.
+//
+// Mathematical layout:
+// STEP_SIZE = 1 << STEP_SIZE_BITS
+// NUM_STEPS = 1 << NUM_STEP_BITS
+// EXP_BASE = STEP_SIZE * NUM_STEPS
+// LARGE_SIZE_THRESHOLD = UNIT_SIZE * EXP_BASE
+//
+// Visual representation with example parameters:
+// UNIT_SIZE = 32, STEP_SIZE = 8, NUM_STEPS = 4
+// EXP_BASE = 32, THRESHOLD = 1024 B (1 KiB)
+//
+// 1. Small Sizes (Linear Array):
+// Covers [0, ... 1024 B] growing directly by UNIT_SIZE = 32 B
+// +-------+-------+-------+-------+-------+-----------+---------------+
+// | [0 B] | [32B] | [64B] | [96B] | ... | [992 B] | [1024 B (Th)] |
+// +-------+-------+-------+-------+-------+-----------+---------------+
+//
+// 2. Large Sizes (2-D Table):
+// Rows = FL (Exponential growth), Columns = SL (Linear steps)
+// One can think of each Row containing NUM_STEPS (4) lists.
+//
+// LINEAR INCREASE ALONG COLUMN (SL) --->
+// +---------------+---------------+---------------+---------------+
+// | Col = 0 | Col = 1 | Col = 2 | Col = 3 |
+// | (Base) | (+25% FL) | (+50% FL) | (+75% FL) |
+// +---------+---------------+---------------+---------------+---------------+
+// E | Row = 0 | 1024 B | 1280 B | 1536 B | 1792 B |
+// X |(Base 1K)| [1024 - 1279] | [1280 - 1535] | [1536 - 1791] | [1792 - 2047] |
+// P +---------+---------------+---------------+---------------+---------------+
+// O | Row = 1 | 2048 B | 2560 B | 3072 B | 3584 B |
+// N |(Base 2K)| [2048 - 2559] | [2560 - 3071] | [3072 - 3583] | [3584 - 4095] |
+// E +---------+---------------+---------------+---------------+---------------+
+// N | Row = 2 | 4096 B | 5120 B | 6144 B | 7168 B |
+// T |(Base 4K)| [4096 - 5119] | [5120 - 6143] | [6144 - 7167] | [7168 - 8191] |
+// I +---------+---------------+---------------+---------------+---------------+
+// A | Row = 3 | 8192 B | 10240 B | 12288 B | 14336 B |
+// L |(Base 8K)|[8192 - 10239]|[10240 - 12287]|[12288 - 14335]|[14336 - 16383]|
+// +---------+---------------+---------------+---------------+---------------+
+//
+// Note: For the real implementation, we don't actually store the lists in a
+// 2-D structure. Instead, we flatten the entire 2-D layout into a single
+// flat 1-D array of size TOTAL_BITS (free_lists), and map sizes directly to
+// a continuous 1-D index using size_to_bit_index. The allocation state is
+// tracked compactly in the lookup_table bitmask array.
+template <typename CONFIG> class TLSFFreeStoreImpl {
+protected:
+ static_assert(cpp::has_single_bit(CONFIG::UNIT_SIZE),
+ "unit size must be a power of two");
+ static_assert(CONFIG::NUM_TABLE_ENTRIES > 0,
+ "the lookup table must have at least one entry");
- /// Remove a best-fit free block that can contain the given size when
- /// allocated. Returns nullptr if there is no such block.
- BlockRef remove_best_fit(size_t size);
+ static constexpr size_t STEP_SIZE = size_t(1) << CONFIG::STEP_SIZE_BITS;
+ static constexpr size_t NUM_STEPS = size_t(1) << CONFIG::NUM_STEP_BITS;
+ static constexpr size_t EXP_BASE = STEP_SIZE * NUM_STEPS;
+ static constexpr int UNIT_SIZE_LOG2 = cpp::bit_width(CONFIG::UNIT_SIZE) - 1;
+ static constexpr int EXP_BASE_LOG2 =
+ CONFIG::STEP_SIZE_BITS + CONFIG::NUM_STEP_BITS;
+ static constexpr size_t BITS_PER_ENTRY =
+ cpp::numeric_limits<uintptr_t>::digits;
+ static constexpr size_t TOTAL_BITS =
+ CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
+ static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_OVERFLOW_BIN;
+ static constexpr size_t OVERFLOW_WIDTH =
+ size_t(1) << (cpp::numeric_limits<size_t>::digits - 2);
-private:
+public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
- static constexpr size_t MIN_LARGE_OUTER_SIZE = align_up(
- BlockRef::HEADER_SIZE + sizeof(FreeTrie::Node), BlockRef::MIN_ALIGN);
- static constexpr size_t NUM_SMALL_SIZES =
- (MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / BlockRef::MIN_ALIGN;
+ LIBC_INLINE TLSFFreeStoreImpl() = default;
+ LIBC_INLINE TLSFFreeStoreImpl(const TLSFFreeStoreImpl &other) = delete;
+ LIBC_INLINE TLSFFreeStoreImpl &
+ operator=(const TLSFFreeStoreImpl &other) = delete;
+
+ LIBC_INLINE void insert(BlockRef block);
+ LIBC_INLINE void remove(BlockRef block);
+ LIBC_INLINE BlockRef remove_best_fit(size_t size) {
+ return find_and_remove_fit(size);
+ }
+ LIBC_INLINE BlockRef find_and_remove_fit(size_t size);
+
+protected:
LIBC_INLINE static bool too_small(BlockRef block) {
return block.outer_size() < MIN_OUTER_SIZE;
}
- LIBC_INLINE static bool is_small(BlockRef block) {
- return block.outer_size() < MIN_LARGE_OUTER_SIZE;
- }
- FreeList &small_list(BlockRef block);
- FreeList *find_best_small_fit(size_t size);
+ union ListOrTrie {
+ FreeList list;
+ FreeTrie::Node *trie_root;
- cpp::array<FreeList, NUM_SMALL_SIZES> small_lists;
- FreeTrie large_trie;
+ LIBC_INLINE constexpr ListOrTrie() : trie_root(nullptr) {}
+ };
+
+ cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
+ cpp::array<ListOrTrie, TOTAL_BITS> free_lists{};
+
+ LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
+ LIBC_INLINE void set_bit(size_t bit_index);
+ LIBC_INLINE void clear_bit(size_t bit_index);
+ LIBC_INLINE bool get_bit(size_t bit_index) const;
+ LIBC_INLINE size_t find_first_bit_set_after(size_t bit_index) const;
+ LIBC_INLINE BlockRef remove_first_fit_in_list(size_t index, size_t size);
+ LIBC_INLINE FreeTrie get_trie();
+ LIBC_INLINE void set_trie(const FreeTrie &trie);
+ LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t size);
};
-LIBC_INLINE void FreeStore::insert(BlockRef block) {
+template <typename CONFIG>
+LIBC_INLINE constexpr size_t
+TLSFFreeStoreImpl<CONFIG>::size_to_bit_index(size_t size) {
+ if (size <= (EXP_BASE << UNIT_SIZE_LOG2))
+ return size >> UNIT_SIZE_LOG2;
+
+ size_t size_ilog2 = static_cast<size_t>(cpp::bit_width(size) - 1);
+ size_t exp_offset = (size_ilog2 - UNIT_SIZE_LOG2 - EXP_BASE_LOG2 - 1)
+ << CONFIG::NUM_STEP_BITS;
+ size_t step_index = size >> (size_ilog2 - CONFIG::NUM_STEP_BITS);
+ size_t index = EXP_BASE + exp_offset + step_index;
+
+ return index < TOTAL_BITS ? index : TOTAL_BITS - 1;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::set_bit(size_t bit_index) {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ lookup_table[entry_index] |= uintptr_t(1) << bit_offset;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::clear_bit(size_t bit_index) {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ lookup_table[entry_index] &= ~(uintptr_t(1) << bit_offset);
+}
+
+template <typename CONFIG>
+LIBC_INLINE bool TLSFFreeStoreImpl<CONFIG>::get_bit(size_t bit_index) const {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ return (lookup_table[entry_index] & (uintptr_t(1) << bit_offset)) != 0;
+}
+
+template <typename CONFIG>
+LIBC_INLINE size_t
+TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
+ if (bit_index >= TOTAL_BITS - 1)
+ return TOTAL_BITS;
+
+ size_t target_index = bit_index + 1;
+ size_t start_entry = target_index / BITS_PER_ENTRY;
+ size_t bit_offset = target_index % BITS_PER_ENTRY;
+
+ uintptr_t value = lookup_table[start_entry] & (~uintptr_t(0) << bit_offset);
+ if (value != 0)
+ return start_entry * BITS_PER_ENTRY +
+ static_cast<size_t>(cpp::countr_zero(value));
+
+ for (size_t i = start_entry + 1; i < CONFIG::NUM_TABLE_ENTRIES; ++i) {
+ value = lookup_table[i];
+ if (value != 0)
+ return i * BITS_PER_ENTRY + static_cast<size_t>(cpp::countr_zero(value));
+ }
+ return TOTAL_BITS;
+}
+
+template <typename CONFIG>
+LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
+ return FreeTrie(FreeTrie::SizeRange(0, OVERFLOW_WIDTH),
+ free_lists[TOTAL_BITS - 1].trie_root);
+}
+
+template <typename CONFIG>
+LIBC_INLINE void
+TLSFFreeStoreImpl<CONFIG>::set_trie(const FreeTrie &trie) {
+ free_lists[TOTAL_BITS - 1].trie_root = trie.get_root();
+}
+
+template <typename CONFIG>
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit_in_trie(size_t size) {
+ FreeTrie trie = get_trie();
+ if (FreeTrie::Node *best_fit = trie.find_best_fit(size)) {
+ BlockRef block = best_fit->block();
+ trie.remove(best_fit);
+ set_trie(trie);
+ if (trie.empty())
+ clear_bit(TOTAL_BITS - 1);
+ return block;
+ }
+ return BlockRef();
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
if (too_small(block))
return;
- if (is_small(block))
- small_list(block).push(block);
- else
- large_trie.push(block);
+ size_t bit_index = size_to_bit_index(block.inner_size());
+
+ if constexpr (USE_TRIE)
+ if (bit_index == TOTAL_BITS - 1) {
+ FreeTrie trie = get_trie();
+ trie.push(block);
+ set_trie(trie);
+ set_bit(bit_index);
+ return;
+ }
+
+ free_lists[bit_index].list.push(block);
+ set_bit(bit_index);
}
-LIBC_INLINE void FreeStore::remove(BlockRef block) {
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
if (too_small(block))
return;
- if (is_small(block)) {
- small_list(block).remove(
- reinterpret_cast<FreeList::Node *>(block.usable_space()));
- } else {
- large_trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
- }
+ size_t bit_index = size_to_bit_index(block.inner_size());
+
+ if constexpr (USE_TRIE)
+ if (bit_index == TOTAL_BITS - 1) {
+ FreeTrie trie = get_trie();
+ trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
+ set_trie(trie);
+ if (trie.empty())
+ clear_bit(bit_index);
+ return;
+ }
+
+ free_lists[bit_index].list.remove(
+ reinterpret_cast<FreeList::Node *>(block.usable_space()));
+ if (free_lists[bit_index].list.empty())
+ clear_bit(bit_index);
}
-LIBC_INLINE BlockRef FreeStore::remove_best_fit(size_t size) {
- if (FreeList *list = find_best_small_fit(size)) {
- BlockRef block = list->front();
- list->pop();
- return block;
+template <typename CONFIG>
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
+ FreeList::Node *begin_node = free_lists[index].list.begin();
+ if (begin_node == nullptr)
+ return BlockRef();
+
+ FreeList::Node *cur = begin_node;
+ do {
+ if (cur->size() >= size) {
+ free_lists[index].list.remove(cur);
+ if (free_lists[index].list.empty())
+ clear_bit(index);
+ return cur->block();
+ }
+ cur = cur->next_node();
+ } while (cur != begin_node);
+
+ return BlockRef();
+}
+
+template <typename CONFIG>
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
+ size_t bit_index = size_to_bit_index(size);
+
+ if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1)) {
+ if constexpr (USE_TRIE)
+ return find_and_remove_fit_in_trie(size);
+ else
+ return remove_first_fit_in_list(TOTAL_BITS - 1, size);
}
- if (FreeTrie::Node *best_fit = large_trie.find_best_fit(size)) {
- BlockRef block = best_fit->block();
- large_trie.remove(best_fit);
+
+ // 1. Try oversized bins (guaranteed fit, but larger).
+ size_t oversized_bit = find_first_bit_set_after(bit_index);
+ if (LIBC_LIKELY(oversized_bit < TOTAL_BITS)) {
+ if constexpr (USE_TRIE) {
+ if (oversized_bit == TOTAL_BITS - 1)
+ return find_and_remove_fit_in_trie(size);
+ }
+
+ BlockRef block = free_lists[oversized_bit].list.front();
+ free_lists[oversized_bit].list.pop();
+ if (free_lists[oversized_bit].list.empty())
+ clear_bit(oversized_bit);
return block;
}
+
+ // 2. Try exact fit (fallback).
+ if (get_bit(bit_index)) {
+ if (BlockRef block = remove_first_fit_in_list(bit_index, size))
+ return block;
+ }
+
return BlockRef();
}
-LIBC_INLINE FreeList &FreeStore::small_list(BlockRef block) {
- LIBC_ASSERT(is_small(block) && "only legal for small blocks");
- return small_lists[(block.outer_size() - MIN_OUTER_SIZE) /
- BlockRef::MIN_ALIGN];
-}
+template <size_t UNIT_SIZE, size_t STEP_SIZE_BITS, size_t NUM_STEP_BITS,
+ size_t NUM_TABLE_ENTRIES, bool USE_TRIE = false>
+using TLSFFreeStore = TLSFFreeStoreImpl<TLSFFreeStoreConfig<
+ UNIT_SIZE, STEP_SIZE_BITS, NUM_STEP_BITS, NUM_TABLE_ENTRIES, USE_TRIE>>;
-LIBC_INLINE FreeList *FreeStore::find_best_small_fit(size_t size) {
- for (FreeList &list : small_lists)
- if (!list.empty() && list.size() >= size)
- return &list;
- return nullptr;
-}
+#ifndef LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN
+#define LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN false
+#endif
+
+using FreeStore =
+ TLSFFreeStore<BlockRef::MIN_ALIGN, 3, 2, (sizeof(uintptr_t) == 8 ? 3 : 6),
+ LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN>;
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 9e35463462b38..728d1ad4f294f 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -88,7 +88,8 @@ class FreeTrie {
};
LIBC_INLINE constexpr FreeTrie() : FreeTrie(SizeRange{0, 0}) {}
- LIBC_INLINE constexpr FreeTrie(SizeRange range) : range(range) {}
+ LIBC_INLINE constexpr FreeTrie(SizeRange range, Node *root = nullptr)
+ : root(root), range(range) {}
/// Sets the range of possible block sizes. This can only be called when the
/// trie is empty.
@@ -100,6 +101,9 @@ class FreeTrie {
/// @returns Whether the trie contains any blocks.
LIBC_INLINE bool empty() const { return !root; }
+ /// @returns The root node of the trie.
+ LIBC_INLINE Node *get_root() const { return root; }
+
/// Push a block to the trie.
void push(BlockRef block);
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index 61103a9126c08..ce655367da00f 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -38,7 +38,6 @@ TEST(LlvmLibcFreeStore, TooSmall) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(too_small);
store.insert(remainder);
@@ -68,21 +67,26 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(smallest);
if (largest_small != smallest)
store.insert(largest_small);
store.insert(remainder);
- // Find exact match for smallest.
- ASSERT_EQ(store.remove_best_fit(smallest.inner_size()).addr(),
- smallest.addr());
- store.insert(smallest);
-
- // Find exact match for largest.
- ASSERT_EQ(store.remove_best_fit(largest_small.inner_size()).addr(),
- largest_small.addr());
- store.insert(largest_small);
+ // For TLSF (oversized first), asking for a size will return the block from
+ // the first non-empty oversized bin if one exists, bypassing the exact bin.
+ if (largest_small != smallest) {
+ BlockRef block = store.remove_best_fit(smallest.inner_size());
+ ASSERT_EQ(block.addr(), largest_small.addr());
+ store.insert(block);
+
+ BlockRef block2 = store.remove_best_fit(largest_small.inner_size());
+ ASSERT_EQ(block2.addr(), remainder.addr());
+ store.insert(block2);
+ } else {
+ BlockRef block = store.remove_best_fit(smallest.inner_size());
+ ASSERT_EQ(block.addr(), remainder.addr());
+ store.insert(block);
+ }
// Search small list for best fit.
BlockRef next_smallest =
@@ -108,7 +112,6 @@ TEST(LlvmLibcFreeStore, Remove) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(small);
store.insert(remainder);
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index bf3284b2faf64..17a81fa41e9a1 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -132,3 +132,19 @@ TEST(LlvmLibcFreeTrie, Remove) {
EXPECT_EQ(trie.find_best_fit(large.inner_size())->block().addr(),
large.addr());
}
+
+TEST(LlvmLibcFreeTrie, ConstructorWithRoot) {
+ FreeTrie::Node *root = nullptr;
+ FreeTrie trie({0, 4096}, root);
+ EXPECT_TRUE(trie.empty());
+ EXPECT_EQ(trie.get_root(), static_cast<FreeTrie::Node *>(nullptr));
+
+ byte mem[1024];
+ optional<BlockRef> maybeBlock = BlockRef::init(mem);
+ ASSERT_TRUE(maybeBlock.has_value());
+ BlockRef block = *maybeBlock;
+ trie.push(block);
+
+ EXPECT_FALSE(trie.empty());
+ EXPECT_NE(trie.get_root(), static_cast<FreeTrie::Node *>(nullptr));
+}
>From 2565f7f4075e62ffbdee69c8bfa59e56a36a9f3a Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:38:44 -0700
Subject: [PATCH 02/12] [libc] clean up extra changes
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freelist.cpp | 3 -
libc/src/__support/freelist_heap.h | 1 +
libc/src/__support/freestore.h | 36 ++++++++++--
libc/test/src/__support/freestore_test.cpp | 68 +++++++++++++---------
4 files changed, 74 insertions(+), 34 deletions(-)
diff --git a/libc/src/__support/freelist.cpp b/libc/src/__support/freelist.cpp
index 2fdcaadbdb554..2f468b08e4f7a 100644
--- a/libc/src/__support/freelist.cpp
+++ b/libc/src/__support/freelist.cpp
@@ -17,9 +17,6 @@ namespace LIBC_NAMESPACE_DECL {
void FreeList::push(Node *node) {
if (begin_) {
- LIBC_ASSERT(BlockRef::from_usable_space(node).outer_size() ==
- begin_->block().outer_size() &&
- "freelist entries must have the same size");
// Since the list is circular, insert the node immediately before begin_.
node->prev = begin_->prev;
node->next = begin_;
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index d8d937eba50d8..73a80754050dd 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -85,6 +85,7 @@ LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
+ free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
free_store.insert(block);
is_initialized = true;
}
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 9394972f3adcc..d72c9c6308c5a 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -109,8 +109,6 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
static constexpr size_t TOTAL_BITS =
CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_OVERFLOW_BIN;
- static constexpr size_t OVERFLOW_WIDTH =
- size_t(1) << (cpp::numeric_limits<size_t>::digits - 2);
public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
@@ -121,6 +119,8 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
LIBC_INLINE TLSFFreeStoreImpl &
operator=(const TLSFFreeStoreImpl &other) = delete;
+ LIBC_INLINE static constexpr size_t index_to_min_size(size_t index);
+ LIBC_INLINE void set_range(FreeTrie::SizeRange range);
LIBC_INLINE void insert(BlockRef block);
LIBC_INLINE void remove(BlockRef block);
LIBC_INLINE BlockRef remove_best_fit(size_t size) {
@@ -142,6 +142,7 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
cpp::array<ListOrTrie, TOTAL_BITS> free_lists{};
+ FreeTrie::SizeRange trie_range{index_to_min_size(TOTAL_BITS - 1), 1};
LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
LIBC_INLINE void set_bit(size_t bit_index);
@@ -213,10 +214,37 @@ TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
return TOTAL_BITS;
}
+template <typename CONFIG>
+LIBC_INLINE constexpr size_t
+TLSFFreeStoreImpl<CONFIG>::index_to_min_size(size_t index) {
+ if (index <= EXP_BASE)
+ return index << UNIT_SIZE_LOG2;
+
+ size_t local_index = index - EXP_BASE;
+ size_t exp_index = local_index >> CONFIG::NUM_STEP_BITS;
+ size_t linear_index = local_index & (NUM_STEPS - 1);
+
+ size_t row_base = (EXP_BASE << exp_index) << UNIT_SIZE_LOG2;
+ size_t step_size = (STEP_SIZE << exp_index) << UNIT_SIZE_LOG2;
+ return row_base + linear_index * step_size;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void
+TLSFFreeStoreImpl<CONFIG>::set_range(FreeTrie::SizeRange range) {
+ if constexpr (USE_TRIE) {
+ size_t heap_max = range.min + range.width;
+ size_t overflow_min = index_to_min_size(TOTAL_BITS - 1);
+ size_t width = 1;
+ if (heap_max > overflow_min)
+ width = cpp::bit_ceil(heap_max - overflow_min);
+ trie_range = FreeTrie::SizeRange(overflow_min, width);
+ }
+}
+
template <typename CONFIG>
LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
- return FreeTrie(FreeTrie::SizeRange(0, OVERFLOW_WIDTH),
- free_lists[TOTAL_BITS - 1].trie_root);
+ return FreeTrie(trie_range, free_lists[TOTAL_BITS - 1].trie_root);
}
template <typename CONFIG>
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index ce655367da00f..d56ed96143934 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -38,6 +38,7 @@ TEST(LlvmLibcFreeStore, TooSmall) {
BlockRef remainder = *maybeBlock;
FreeStore store;
+ store.set_range({0, 4096});
store.insert(too_small);
store.insert(remainder);
@@ -46,7 +47,7 @@ TEST(LlvmLibcFreeStore, TooSmall) {
store.remove(too_small);
}
-TEST(LlvmLibcFreeStore, RemoveBestFit) {
+TEST(LlvmLibcFreeStore, RemoveFit) {
byte mem[1024];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
@@ -67,37 +68,34 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) {
BlockRef remainder = *maybeBlock;
FreeStore store;
+ store.set_range({0, 4096});
store.insert(smallest);
if (largest_small != smallest)
store.insert(largest_small);
store.insert(remainder);
- // For TLSF (oversized first), asking for a size will return the block from
- // the first non-empty oversized bin if one exists, bypassing the exact bin.
- if (largest_small != smallest) {
- BlockRef block = store.remove_best_fit(smallest.inner_size());
- ASSERT_EQ(block.addr(), largest_small.addr());
- store.insert(block);
-
- BlockRef block2 = store.remove_best_fit(largest_small.inner_size());
- ASSERT_EQ(block2.addr(), remainder.addr());
- store.insert(block2);
- } else {
- BlockRef block = store.remove_best_fit(smallest.inner_size());
- ASSERT_EQ(block.addr(), remainder.addr());
- store.insert(block);
- }
-
- // Search small list for best fit.
- BlockRef next_smallest =
- largest_small == smallest ? remainder : largest_small;
- ASSERT_EQ(store.remove_best_fit(smallest.inner_size() + 1).addr(),
- next_smallest.addr());
- store.insert(next_smallest);
-
- // Continue search for best fit to large blocks.
- EXPECT_EQ(store.remove_best_fit(largest_small.inner_size() + 1).addr(),
- remainder.addr());
+ // Requesting smallest size returns a valid block fitting the size.
+ BlockRef block1 = store.remove_best_fit(smallest.inner_size());
+ ASSERT_NE(block1.addr(), BlockRef().addr());
+ ASSERT_GE(block1.inner_size(), smallest.inner_size());
+ store.insert(block1);
+
+ // Requesting largest_small size returns a valid block fitting the size.
+ BlockRef block2 = store.remove_best_fit(largest_small.inner_size());
+ ASSERT_NE(block2.addr(), BlockRef().addr());
+ ASSERT_GE(block2.inner_size(), largest_small.inner_size());
+ store.insert(block2);
+
+ // Requesting smallest inner_size + 1 returns a valid block fitting the size.
+ BlockRef block3 = store.remove_best_fit(smallest.inner_size() + 1);
+ ASSERT_NE(block3.addr(), BlockRef().addr());
+ ASSERT_GE(block3.inner_size(), smallest.inner_size() + 1);
+ store.insert(block3);
+
+ // Requesting largest_small inner_size + 1 returns a valid block fitting the size.
+ BlockRef block4 = store.remove_best_fit(largest_small.inner_size() + 1);
+ ASSERT_NE(block4.addr(), BlockRef().addr());
+ ASSERT_GE(block4.inner_size(), largest_small.inner_size() + 1);
}
TEST(LlvmLibcFreeStore, Remove) {
@@ -112,6 +110,7 @@ TEST(LlvmLibcFreeStore, Remove) {
BlockRef remainder = *maybeBlock;
FreeStore store;
+ store.set_range({0, 4096});
store.insert(small);
store.insert(remainder);
@@ -122,3 +121,18 @@ TEST(LlvmLibcFreeStore, Remove) {
ASSERT_EQ(store.remove_best_fit(small.inner_size()).addr(),
BlockRef().addr());
}
+
+TEST(LlvmLibcFreeStore, IndexToMinSize) {
+ constexpr size_t min_size_0 = FreeStore::index_to_min_size(0);
+ EXPECT_EQ(min_size_0, static_cast<size_t>(0));
+
+ constexpr size_t min_size_1 = FreeStore::index_to_min_size(1);
+ EXPECT_EQ(min_size_1, static_cast<size_t>(BlockRef::MIN_ALIGN));
+
+ size_t prev_size = 0;
+ for (size_t i = 1; i < 64; ++i) {
+ size_t min_size = FreeStore::index_to_min_size(i);
+ EXPECT_GT(min_size, prev_size);
+ prev_size = min_size;
+ }
+}
>From 48c586d949b22465e614311d812ac413b7d74698 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:41:00 -0700
Subject: [PATCH 03/12] [libc] rename next_node to next and get_root to root
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freelist.cpp | 18 +++++++++---------
libc/src/__support/freelist.h | 6 +++---
libc/src/__support/freestore.h | 4 ++--
libc/src/__support/freetrie.cpp | 4 ++--
libc/src/__support/freetrie.h | 14 +++++++-------
libc/test/src/__support/freetrie_test.cpp | 4 ++--
6 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/libc/src/__support/freelist.cpp b/libc/src/__support/freelist.cpp
index 2f468b08e4f7a..c0834fe6029d8 100644
--- a/libc/src/__support/freelist.cpp
+++ b/libc/src/__support/freelist.cpp
@@ -18,26 +18,26 @@ namespace LIBC_NAMESPACE_DECL {
void FreeList::push(Node *node) {
if (begin_) {
// Since the list is circular, insert the node immediately before begin_.
- node->prev = begin_->prev;
- node->next = begin_;
- begin_->prev->next = node;
- begin_->prev = node;
+ node->prev_ = begin_->prev_;
+ node->next_ = begin_;
+ begin_->prev_->next_ = node;
+ begin_->prev_ = node;
} else {
- begin_ = node->prev = node->next = node;
+ begin_ = node->prev_ = node->next_ = node;
}
}
void FreeList::remove(Node *node) {
LIBC_ASSERT(begin_ && "cannot remove from empty list");
- Node *next = node->next;
+ Node *next = node->next_;
if (node == next) {
LIBC_ASSERT(node == begin_ &&
"a self-referential node must be the only element");
begin_ = nullptr;
} else {
- Node *prev = node->prev;
- prev->next = next;
- next->prev = prev;
+ Node *prev = node->prev_;
+ prev->next_ = next;
+ next->prev_ = prev;
if (begin_ == node)
begin_ = next;
}
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index ae2de684b3a24..4bac60336761f 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -42,12 +42,12 @@ class FreeList {
LIBC_INLINE size_t size() const { return block().inner_size(); }
/// @returns The next node in the list containing this node.
- LIBC_INLINE Node *next_node() const { return next; }
+ LIBC_INLINE Node *next() const { return next_; }
private:
// Circularly linked pointers to adjacent nodes.
- Node *prev;
- Node *next;
+ Node *prev_;
+ Node *next_;
friend class FreeList;
};
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index d72c9c6308c5a..6f832100bd1d4 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -250,7 +250,7 @@ LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
template <typename CONFIG>
LIBC_INLINE void
TLSFFreeStoreImpl<CONFIG>::set_trie(const FreeTrie &trie) {
- free_lists[TOTAL_BITS - 1].trie_root = trie.get_root();
+ free_lists[TOTAL_BITS - 1].trie_root = trie.root();
}
template <typename CONFIG>
@@ -324,7 +324,7 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
clear_bit(index);
return cur->block();
}
- cur = cur->next_node();
+ cur = cur->next();
} while (cur != begin_node);
return BlockRef();
diff --git a/libc/src/__support/freetrie.cpp b/libc/src/__support/freetrie.cpp
index e76efe717f215..e3342080b623a 100644
--- a/libc/src/__support/freetrie.cpp
+++ b/libc/src/__support/freetrie.cpp
@@ -52,8 +52,8 @@ void FreeTrie::replace_node(Node *node, Node *new_node) {
"no reference to child node found in parent");
parent_child = new_node;
} else {
- LIBC_ASSERT(root == node && "non-root node had no parent");
- root = new_node;
+ LIBC_ASSERT(root_ == node && "non-root node had no parent");
+ root_ = new_node;
}
if (node->lower)
node->lower->parent = new_node;
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 728d1ad4f294f..9e0d61b350065 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -89,7 +89,7 @@ class FreeTrie {
LIBC_INLINE constexpr FreeTrie() : FreeTrie(SizeRange{0, 0}) {}
LIBC_INLINE constexpr FreeTrie(SizeRange range, Node *root = nullptr)
- : root(root), range(range) {}
+ : root_(root), range(range) {}
/// Sets the range of possible block sizes. This can only be called when the
/// trie is empty.
@@ -99,10 +99,10 @@ class FreeTrie {
}
/// @returns Whether the trie contains any blocks.
- LIBC_INLINE bool empty() const { return !root; }
+ LIBC_INLINE bool empty() const { return !root_; }
/// @returns The root node of the trie.
- LIBC_INLINE Node *get_root() const { return root; }
+ LIBC_INLINE Node *root() const { return root_; }
/// Push a block to the trie.
void push(BlockRef block);
@@ -116,13 +116,13 @@ class FreeTrie {
private:
/// @returns Whether a node is the head of its containing freelist.
- bool is_head(Node *node) const { return node->parent || node == root; }
+ bool is_head(Node *node) const { return node->parent || node == root_; }
/// Replaces references to one node with another (or nullptr) in all adjacent
/// parent and child nodes.
void replace_node(Node *node, Node *new_node);
- Node *root = nullptr;
+ Node *root_ = nullptr;
SizeRange range;
};
@@ -133,7 +133,7 @@ LIBC_INLINE void FreeTrie::push(BlockRef block) {
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
// Find the position in the tree to push to.
- Node **cur = &root;
+ Node **cur = &root_;
Node *parent = nullptr;
SizeRange cur_range = range;
while (*cur && (*cur)->size() != size) {
@@ -164,7 +164,7 @@ LIBC_INLINE FreeTrie::Node *FreeTrie::find_best_fit(size_t size) {
if (empty() || range.max() < size)
return nullptr;
- Node *cur = root;
+ Node *cur = root_;
SizeRange cur_range = range;
Node *best_fit = nullptr;
Node *deferred_upper_trie = nullptr;
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 17a81fa41e9a1..267c6eb7d8894 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -137,7 +137,7 @@ TEST(LlvmLibcFreeTrie, ConstructorWithRoot) {
FreeTrie::Node *root = nullptr;
FreeTrie trie({0, 4096}, root);
EXPECT_TRUE(trie.empty());
- EXPECT_EQ(trie.get_root(), static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_EQ(trie.root(), static_cast<FreeTrie::Node *>(nullptr));
byte mem[1024];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
@@ -146,5 +146,5 @@ TEST(LlvmLibcFreeTrie, ConstructorWithRoot) {
trie.push(block);
EXPECT_FALSE(trie.empty());
- EXPECT_NE(trie.get_root(), static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_NE(trie.root(), static_cast<FreeTrie::Node *>(nullptr));
}
>From 6a2662c18ea298e824790cc331e31fa06da4389a Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:45:54 -0700
Subject: [PATCH 04/12] [libc] refactor DefaultFreeStoreConfig for
TLSFFreeStore
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freestore.h | 32 ++++++++++----------------------
1 file changed, 10 insertions(+), 22 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 6f832100bd1d4..4ccf57c505595 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -27,17 +27,13 @@
namespace LIBC_NAMESPACE_DECL {
-/// Configuration for TLSFFreeStore.
-template <size_t UNIT_SIZE_VAL, size_t STEP_SIZE_BITS_VAL,
- size_t NUM_STEP_BITS_VAL, size_t NUM_TABLE_ENTRIES_VAL,
- bool USE_TRIE_FOR_OVERFLOW_BIN_VAL = false>
-struct TLSFFreeStoreConfig {
- static constexpr size_t UNIT_SIZE = UNIT_SIZE_VAL;
- static constexpr size_t STEP_SIZE_BITS = STEP_SIZE_BITS_VAL;
- static constexpr size_t NUM_STEP_BITS = NUM_STEP_BITS_VAL;
- static constexpr size_t NUM_TABLE_ENTRIES = NUM_TABLE_ENTRIES_VAL;
- static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN =
- USE_TRIE_FOR_OVERFLOW_BIN_VAL;
+/// Default configuration for TLSFFreeStore.
+struct DefaultFreeStoreConfig {
+ static constexpr size_t UNIT_SIZE = BlockRef::MIN_ALIGN;
+ static constexpr size_t STEP_SIZE_BITS = 3;
+ static constexpr size_t NUM_STEP_BITS = 2;
+ static constexpr size_t NUM_TABLE_ENTRIES = sizeof(uintptr_t) == 8 ? 3 : 6;
+ static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN = true;
};
// A two-level segregated fit store for free blocks.
@@ -366,18 +362,10 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
return BlockRef();
}
-template <size_t UNIT_SIZE, size_t STEP_SIZE_BITS, size_t NUM_STEP_BITS,
- size_t NUM_TABLE_ENTRIES, bool USE_TRIE = false>
-using TLSFFreeStore = TLSFFreeStoreImpl<TLSFFreeStoreConfig<
- UNIT_SIZE, STEP_SIZE_BITS, NUM_STEP_BITS, NUM_TABLE_ENTRIES, USE_TRIE>>;
+template <typename CONFIG = DefaultFreeStoreConfig>
+using TLSFFreeStore = TLSFFreeStoreImpl<CONFIG>;
-#ifndef LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN
-#define LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN false
-#endif
-
-using FreeStore =
- TLSFFreeStore<BlockRef::MIN_ALIGN, 3, 2, (sizeof(uintptr_t) == 8 ? 3 : 6),
- LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN>;
+using FreeStore = TLSFFreeStore<DefaultFreeStoreConfig>;
} // namespace LIBC_NAMESPACE_DECL
>From b50021a5a6da66105d0cc8d40f2008550e385cf7 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:47:43 -0700
Subject: [PATCH 05/12] [libc] introduce LINEAR_SCAN_LIMIT to FreeStore config
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freestore.h | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 4ccf57c505595..721abb244985a 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -34,6 +34,7 @@ struct DefaultFreeStoreConfig {
static constexpr size_t NUM_STEP_BITS = 2;
static constexpr size_t NUM_TABLE_ENTRIES = sizeof(uintptr_t) == 8 ? 3 : 6;
static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN = true;
+ static constexpr size_t LINEAR_SCAN_LIMIT = 16;
};
// A two-level segregated fit store for free blocks.
@@ -313,6 +314,7 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
return BlockRef();
FreeList::Node *cur = begin_node;
+ size_t count = 0;
do {
if (cur->size() >= size) {
free_lists[index].list.remove(cur);
@@ -321,7 +323,8 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
return cur->block();
}
cur = cur->next();
- } while (cur != begin_node);
+ ++count;
+ } while (cur != begin_node && count < CONFIG::LINEAR_SCAN_LIMIT);
return BlockRef();
}
>From ad665d6a42db610da3d9a06e78da638085fb08d5 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:50:48 -0700
Subject: [PATCH 06/12] [libc] replace union ListOrTrie with struct using
uintptr_t payload and bit_cast
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freestore.h | 48 +++++++++++++++++++++++-----------
1 file changed, 33 insertions(+), 15 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 721abb244985a..e4ac5d193d7c3 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -130,11 +130,21 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
return block.outer_size() < MIN_OUTER_SIZE;
}
- union ListOrTrie {
- FreeList list;
- FreeTrie::Node *trie_root;
+ struct ListOrTrie {
+ uintptr_t payload = 0;
- LIBC_INLINE constexpr ListOrTrie() : trie_root(nullptr) {}
+ LIBC_INLINE FreeList list() const {
+ return FreeList(cpp::bit_cast<FreeList::Node *>(payload));
+ }
+ LIBC_INLINE void set_list(FreeList l) {
+ payload = cpp::bit_cast<uintptr_t>(l.begin());
+ }
+ LIBC_INLINE FreeTrie::Node *trie_root() const {
+ return cpp::bit_cast<FreeTrie::Node *>(payload);
+ }
+ LIBC_INLINE void set_trie_root(FreeTrie::Node *r) {
+ payload = cpp::bit_cast<uintptr_t>(r);
+ }
};
cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
@@ -241,13 +251,13 @@ TLSFFreeStoreImpl<CONFIG>::set_range(FreeTrie::SizeRange range) {
template <typename CONFIG>
LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
- return FreeTrie(trie_range, free_lists[TOTAL_BITS - 1].trie_root);
+ return FreeTrie(trie_range, free_lists[TOTAL_BITS - 1].trie_root());
}
template <typename CONFIG>
LIBC_INLINE void
TLSFFreeStoreImpl<CONFIG>::set_trie(const FreeTrie &trie) {
- free_lists[TOTAL_BITS - 1].trie_root = trie.root();
+ free_lists[TOTAL_BITS - 1].set_trie_root(trie.root());
}
template <typename CONFIG>
@@ -280,7 +290,9 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
return;
}
- free_lists[bit_index].list.push(block);
+ FreeList list = free_lists[bit_index].list();
+ list.push(block);
+ free_lists[bit_index].set_list(list);
set_bit(bit_index);
}
@@ -300,16 +312,19 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
return;
}
- free_lists[bit_index].list.remove(
+ FreeList list = free_lists[bit_index].list();
+ list.remove(
reinterpret_cast<FreeList::Node *>(block.usable_space()));
- if (free_lists[bit_index].list.empty())
+ if (list.empty())
clear_bit(bit_index);
+ free_lists[bit_index].set_list(list);
}
template <typename CONFIG>
LIBC_INLINE BlockRef
TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
- FreeList::Node *begin_node = free_lists[index].list.begin();
+ FreeList list = free_lists[index].list();
+ FreeList::Node *begin_node = list.begin();
if (begin_node == nullptr)
return BlockRef();
@@ -317,9 +332,10 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
size_t count = 0;
do {
if (cur->size() >= size) {
- free_lists[index].list.remove(cur);
- if (free_lists[index].list.empty())
+ list.remove(cur);
+ if (list.empty())
clear_bit(index);
+ free_lists[index].set_list(list);
return cur->block();
}
cur = cur->next();
@@ -349,10 +365,12 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
return find_and_remove_fit_in_trie(size);
}
- BlockRef block = free_lists[oversized_bit].list.front();
- free_lists[oversized_bit].list.pop();
- if (free_lists[oversized_bit].list.empty())
+ FreeList list = free_lists[oversized_bit].list();
+ BlockRef block = list.front();
+ list.pop();
+ if (list.empty())
clear_bit(oversized_bit);
+ free_lists[oversized_bit].set_list(list);
return block;
}
>From 2d20d7392609d0869ebf5578a25713c49fd9826a Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:52:17 -0700
Subject: [PATCH 07/12] [libc] simplify TLSFFreeStoreImpl with
free_lists[BITS-1] and FreeTrie trie members
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freestore.h | 95 ++++++++++++----------------------
1 file changed, 32 insertions(+), 63 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index e4ac5d193d7c3..489218c584ab3 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -130,26 +130,10 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
return block.outer_size() < MIN_OUTER_SIZE;
}
- struct ListOrTrie {
- uintptr_t payload = 0;
-
- LIBC_INLINE FreeList list() const {
- return FreeList(cpp::bit_cast<FreeList::Node *>(payload));
- }
- LIBC_INLINE void set_list(FreeList l) {
- payload = cpp::bit_cast<uintptr_t>(l.begin());
- }
- LIBC_INLINE FreeTrie::Node *trie_root() const {
- return cpp::bit_cast<FreeTrie::Node *>(payload);
- }
- LIBC_INLINE void set_trie_root(FreeTrie::Node *r) {
- payload = cpp::bit_cast<uintptr_t>(r);
- }
- };
-
cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
- cpp::array<ListOrTrie, TOTAL_BITS> free_lists{};
- FreeTrie::SizeRange trie_range{index_to_min_size(TOTAL_BITS - 1), 1};
+ cpp::array<FreeList, TOTAL_BITS - 1> free_lists{};
+ FreeTrie trie{};
+ FreeList overflow_list{};
LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
LIBC_INLINE void set_bit(size_t bit_index);
@@ -157,8 +141,6 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
LIBC_INLINE bool get_bit(size_t bit_index) const;
LIBC_INLINE size_t find_first_bit_set_after(size_t bit_index) const;
LIBC_INLINE BlockRef remove_first_fit_in_list(size_t index, size_t size);
- LIBC_INLINE FreeTrie get_trie();
- LIBC_INLINE void set_trie(const FreeTrie &trie);
LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t size);
};
@@ -245,29 +227,16 @@ TLSFFreeStoreImpl<CONFIG>::set_range(FreeTrie::SizeRange range) {
size_t width = 1;
if (heap_max > overflow_min)
width = cpp::bit_ceil(heap_max - overflow_min);
- trie_range = FreeTrie::SizeRange(overflow_min, width);
+ trie.set_range(FreeTrie::SizeRange(overflow_min, width));
}
}
-template <typename CONFIG>
-LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
- return FreeTrie(trie_range, free_lists[TOTAL_BITS - 1].trie_root());
-}
-
-template <typename CONFIG>
-LIBC_INLINE void
-TLSFFreeStoreImpl<CONFIG>::set_trie(const FreeTrie &trie) {
- free_lists[TOTAL_BITS - 1].set_trie_root(trie.root());
-}
-
template <typename CONFIG>
LIBC_INLINE BlockRef
TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit_in_trie(size_t size) {
- FreeTrie trie = get_trie();
if (FreeTrie::Node *best_fit = trie.find_best_fit(size)) {
BlockRef block = best_fit->block();
trie.remove(best_fit);
- set_trie(trie);
if (trie.empty())
clear_bit(TOTAL_BITS - 1);
return block;
@@ -281,18 +250,16 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
return;
size_t bit_index = size_to_bit_index(block.inner_size());
- if constexpr (USE_TRIE)
- if (bit_index == TOTAL_BITS - 1) {
- FreeTrie trie = get_trie();
+ if (bit_index == TOTAL_BITS - 1) {
+ if constexpr (USE_TRIE)
trie.push(block);
- set_trie(trie);
- set_bit(bit_index);
- return;
- }
+ else
+ overflow_list.push(block);
+ set_bit(bit_index);
+ return;
+ }
- FreeList list = free_lists[bit_index].list();
- list.push(block);
- free_lists[bit_index].set_list(list);
+ free_lists[bit_index].push(block);
set_bit(bit_index);
}
@@ -302,28 +269,31 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
return;
size_t bit_index = size_to_bit_index(block.inner_size());
- if constexpr (USE_TRIE)
- if (bit_index == TOTAL_BITS - 1) {
- FreeTrie trie = get_trie();
+ if (bit_index == TOTAL_BITS - 1) {
+ if constexpr (USE_TRIE) {
trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
- set_trie(trie);
if (trie.empty())
clear_bit(bit_index);
- return;
+ } else {
+ overflow_list.remove(
+ reinterpret_cast<FreeList::Node *>(block.usable_space()));
+ if (overflow_list.empty())
+ clear_bit(bit_index);
}
+ return;
+ }
- FreeList list = free_lists[bit_index].list();
- list.remove(
+ free_lists[bit_index].remove(
reinterpret_cast<FreeList::Node *>(block.usable_space()));
- if (list.empty())
+ if (free_lists[bit_index].empty())
clear_bit(bit_index);
- free_lists[bit_index].set_list(list);
}
template <typename CONFIG>
LIBC_INLINE BlockRef
TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
- FreeList list = free_lists[index].list();
+ FreeList &list =
+ (index == TOTAL_BITS - 1) ? overflow_list : free_lists[index];
FreeList::Node *begin_node = list.begin();
if (begin_node == nullptr)
return BlockRef();
@@ -335,7 +305,6 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
list.remove(cur);
if (list.empty())
clear_bit(index);
- free_lists[index].set_list(list);
return cur->block();
}
cur = cur->next();
@@ -360,17 +329,17 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
// 1. Try oversized bins (guaranteed fit, but larger).
size_t oversized_bit = find_first_bit_set_after(bit_index);
if (LIBC_LIKELY(oversized_bit < TOTAL_BITS)) {
- if constexpr (USE_TRIE) {
- if (oversized_bit == TOTAL_BITS - 1)
+ if (oversized_bit == TOTAL_BITS - 1) {
+ if constexpr (USE_TRIE)
return find_and_remove_fit_in_trie(size);
+ else
+ return remove_first_fit_in_list(TOTAL_BITS - 1, size);
}
- FreeList list = free_lists[oversized_bit].list();
- BlockRef block = list.front();
- list.pop();
- if (list.empty())
+ BlockRef block = free_lists[oversized_bit].front();
+ free_lists[oversized_bit].pop();
+ if (free_lists[oversized_bit].empty())
clear_bit(oversized_bit);
- free_lists[oversized_bit].set_list(list);
return block;
}
>From d40a766df868b3bfac903827ed322ba4f4e0a322 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:59:00 -0700
Subject: [PATCH 08/12] [libc] add fastpath if linear portion are exact
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freestore.h | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 489218c584ab3..82a1f5f2df07e 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -317,6 +317,19 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
template <typename CONFIG>
LIBC_INLINE BlockRef
TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
+ if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
+ size_t index = align_up(size, CONFIG::UNIT_SIZE) >> UNIT_SIZE_LOG2;
+ if (LIBC_LIKELY(index <= EXP_BASE)) {
+ if (get_bit(index)) {
+ BlockRef block = free_lists[index].front();
+ free_lists[index].pop();
+ if (free_lists[index].empty())
+ clear_bit(index);
+ return block;
+ }
+ }
+ }
+
size_t bit_index = size_to_bit_index(size);
if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1)) {
>From 952e967bf390fe955322ee3661642baaa1173887 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 14:32:25 -0700
Subject: [PATCH 09/12] [libc] add fastpath if linear portion are exact
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freestore.h | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 82a1f5f2df07e..c6c3720117185 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -317,16 +317,15 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
template <typename CONFIG>
LIBC_INLINE BlockRef
TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
+ // Fast path for small linear bins if UNIT_SIZE == MIN_ALIGN
if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
size_t index = align_up(size, CONFIG::UNIT_SIZE) >> UNIT_SIZE_LOG2;
- if (LIBC_LIKELY(index <= EXP_BASE)) {
- if (get_bit(index)) {
- BlockRef block = free_lists[index].front();
- free_lists[index].pop();
- if (free_lists[index].empty())
- clear_bit(index);
- return block;
- }
+ if (LIBC_LIKELY(index <= EXP_BASE && get_bit(index))) {
+ BlockRef block = free_lists[index].front();
+ free_lists[index].pop();
+ if (free_lists[index].empty())
+ clear_bit(index);
+ return block;
}
}
@@ -342,7 +341,7 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
// 1. Try oversized bins (guaranteed fit, but larger).
size_t oversized_bit = find_first_bit_set_after(bit_index);
if (LIBC_LIKELY(oversized_bit < TOTAL_BITS)) {
- if (oversized_bit == TOTAL_BITS - 1) {
+ if (LIBC_UNLIKELY(oversized_bit == TOTAL_BITS - 1)) {
if constexpr (USE_TRIE)
return find_and_remove_fit_in_trie(size);
else
>From 572ee3f84d8f4d87f282cd4efd24f583f720fdd5 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Thu, 23 Jul 2026 08:56:45 -0700
Subject: [PATCH 10/12] fmt
---
libc/test/src/__support/freestore_test.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index d56ed96143934..4104af6b6b8a4 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -92,7 +92,8 @@ TEST(LlvmLibcFreeStore, RemoveFit) {
ASSERT_GE(block3.inner_size(), smallest.inner_size() + 1);
store.insert(block3);
- // Requesting largest_small inner_size + 1 returns a valid block fitting the size.
+ // Requesting largest_small inner_size + 1 returns a valid block fitting the
+ // size.
BlockRef block4 = store.remove_best_fit(largest_small.inner_size() + 1);
ASSERT_NE(block4.addr(), BlockRef().addr());
ASSERT_GE(block4.inner_size(), largest_small.inner_size() + 1);
>From b4a77f5fc7c5123ee9966595dcffc48481e30592 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Thu, 30 Jul 2026 14:51:10 -0700
Subject: [PATCH 11/12] [libc][__support] Implement exact linear binning for
TLSFFreeStoreImpl
Previously, size_to_bit_index used size >> UNIT_SIZE_LOG2 for linear bins,
which mapped size 24 to bin 1 and size 17 to bin 1, causing a mismatch
between allocation request sizes and physical block bucket sizes.
This change introduces exact mapping for linear bins:
- Sizes <= MIN_INNER_SIZE map to bin 0.
- Larger linear sizes use ((size - MIN_INNER_SIZE - 1) >> UNIT_SIZE_LOG2) + 1.
- Exponential bins remain unchanged (verified by z3).
TAG=agy
CONV=811f354e-5729-4a5f-af17-de4b5ca9e647
---
libc/src/__support/freestore.h | 35 +++++++++++++++-------
libc/test/src/__support/freestore_test.cpp | 2 +-
2 files changed, 25 insertions(+), 12 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index c6c3720117185..52c6f78e00022 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -110,6 +110,8 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
+ static constexpr size_t MIN_INNER_SIZE =
+ MIN_OUTER_SIZE - BlockRef::HEADER_SIZE + BlockRef::PREV_FIELD_SIZE;
LIBC_INLINE TLSFFreeStoreImpl() = default;
LIBC_INLINE TLSFFreeStoreImpl(const TLSFFreeStoreImpl &other) = delete;
@@ -124,6 +126,7 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
return find_and_remove_fit(size);
}
LIBC_INLINE BlockRef find_and_remove_fit(size_t size);
+ LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
protected:
LIBC_INLINE static bool too_small(BlockRef block) {
@@ -135,7 +138,6 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
FreeTrie trie{};
FreeList overflow_list{};
- LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
LIBC_INLINE void set_bit(size_t bit_index);
LIBC_INLINE void clear_bit(size_t bit_index);
LIBC_INLINE bool get_bit(size_t bit_index) const;
@@ -147,8 +149,14 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
template <typename CONFIG>
LIBC_INLINE constexpr size_t
TLSFFreeStoreImpl<CONFIG>::size_to_bit_index(size_t size) {
- if (size <= (EXP_BASE << UNIT_SIZE_LOG2))
+ if (size <= (EXP_BASE << UNIT_SIZE_LOG2)) {
+ if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
+ if (size <= MIN_INNER_SIZE)
+ return 0;
+ return ((size - MIN_INNER_SIZE - 1) >> UNIT_SIZE_LOG2) + 1;
+ }
return size >> UNIT_SIZE_LOG2;
+ }
size_t size_ilog2 = static_cast<size_t>(cpp::bit_width(size) - 1);
size_t exp_offset = (size_ilog2 - UNIT_SIZE_LOG2 - EXP_BASE_LOG2 - 1)
@@ -206,8 +214,13 @@ TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
template <typename CONFIG>
LIBC_INLINE constexpr size_t
TLSFFreeStoreImpl<CONFIG>::index_to_min_size(size_t index) {
- if (index <= EXP_BASE)
+ if (index <= EXP_BASE) {
+ if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
+ return index == 0 ? 0
+ : MIN_INNER_SIZE + (index - 1) * CONFIG::UNIT_SIZE + 1;
+ }
return index << UNIT_SIZE_LOG2;
+ }
size_t local_index = index - EXP_BASE;
size_t exp_index = local_index >> CONFIG::NUM_STEP_BITS;
@@ -317,20 +330,20 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
template <typename CONFIG>
LIBC_INLINE BlockRef
TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
+
+ size_t bit_index = size_to_bit_index(size);
+
// Fast path for small linear bins if UNIT_SIZE == MIN_ALIGN
if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
- size_t index = align_up(size, CONFIG::UNIT_SIZE) >> UNIT_SIZE_LOG2;
- if (LIBC_LIKELY(index <= EXP_BASE && get_bit(index))) {
- BlockRef block = free_lists[index].front();
- free_lists[index].pop();
- if (free_lists[index].empty())
- clear_bit(index);
+ if (LIBC_LIKELY(bit_index <= EXP_BASE && get_bit(bit_index))) {
+ BlockRef block = free_lists[bit_index].front();
+ free_lists[bit_index].pop();
+ if (free_lists[bit_index].empty())
+ clear_bit(bit_index);
return block;
}
}
- size_t bit_index = size_to_bit_index(size);
-
if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1)) {
if constexpr (USE_TRIE)
return find_and_remove_fit_in_trie(size);
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index 4104af6b6b8a4..e4ee44d2776eb 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -128,7 +128,7 @@ TEST(LlvmLibcFreeStore, IndexToMinSize) {
EXPECT_EQ(min_size_0, static_cast<size_t>(0));
constexpr size_t min_size_1 = FreeStore::index_to_min_size(1);
- EXPECT_EQ(min_size_1, static_cast<size_t>(BlockRef::MIN_ALIGN));
+ EXPECT_EQ(min_size_1, static_cast<size_t>(FreeStore::MIN_INNER_SIZE + 1));
size_t prev_size = 0;
for (size_t i = 1; i < 64; ++i) {
>From 290e9d59155f0e52efce8094dc57babddb34af37 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Fri, 31 Jul 2026 10:22:59 -0700
Subject: [PATCH 12/12] [libc][__support] Fix linear fast path for first
exponential bin in TLSFFreeStoreImpl
Linear bins occupy indices 0 to EXP_BASE - 1. Index EXP_BASE (32) is the first exponential bin.
Previously, index_to_min_size and the linear fast path in find_and_remove_fit used <= EXP_BASE,
which treated index EXP_BASE as a linear bin. This caused find_and_remove_fit to return the
first block in bin EXP_BASE without checking if its size was sufficient for the request.
This change replaces <= with < when comparing against EXP_BASE in index_to_min_size and
find_and_remove_fit, and adds a unit test (NegativeTestForFullHeap).
TAG=agy
CONV=cff84e8c-ee22-4f39-af3c-344d1e6f417b
---
libc/src/__support/freestore.h | 4 ++--
libc/test/src/__support/freelist_heap_test.cpp | 9 +++++++++
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 52c6f78e00022..cc9317dfbb2f1 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -214,7 +214,7 @@ TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
template <typename CONFIG>
LIBC_INLINE constexpr size_t
TLSFFreeStoreImpl<CONFIG>::index_to_min_size(size_t index) {
- if (index <= EXP_BASE) {
+ if (index < EXP_BASE) {
if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
return index == 0 ? 0
: MIN_INNER_SIZE + (index - 1) * CONFIG::UNIT_SIZE + 1;
@@ -335,7 +335,7 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
// Fast path for small linear bins if UNIT_SIZE == MIN_ALIGN
if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
- if (LIBC_LIKELY(bit_index <= EXP_BASE && get_bit(bit_index))) {
+ if (LIBC_LIKELY(bit_index < EXP_BASE && get_bit(bit_index))) {
BlockRef block = free_lists[bit_index].front();
free_lists[bit_index].pop();
if (free_lists[bit_index].empty())
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 1ee6bf0ce4ab4..4f5cdf32cd5f2 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -366,3 +366,12 @@ TEST_FOR_EACH_ALLOCATOR(AllocationSize, 2048) {
allocator.free(ptr);
EXPECT_EQ(allocator.allocation_size(ptr), size_t(0));
}
+
+TEST_FOR_EACH_ALLOCATOR(NegativeTestForFullHeap, 2048) {
+ // Numbers are selected to stress the first exponential bin for 64bit target.
+ void *ptr1 = allocator.allocate(528);
+ allocator.allocate(1000);
+ allocator.free(ptr1);
+ void *ptr3 = allocator.allocate(605);
+ EXPECT_EQ(ptr3, static_cast<void *>(nullptr));
+}
More information about the libc-commits
mailing list