[libc-commits] [libc] [libc] Implement TLSF FreeStore with Trie Overflow Bin (PR #203415)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Mon Aug 3 14:47:40 PDT 2026
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/203415
>From 0c0c9f956e68e981c917822109bdfe83e761f7b5 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:19:19 -0700
Subject: [PATCH 01/14] [libc] Implement TLSF FreeStore
- Replace freestore.h with TLSF implementation.
- Add USE_TRIE_FOR_OVERFLOW_BIN option to use FreeTrie in TLSF.
- Add SizeRange+root constructor and get_root getter to FreeTrie.
- Remove obsolete set_range interface from FreeStore, FreeListHeap, and tests.
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freelist.h | 3 +
libc/src/__support/freelist_heap.h | 1 -
libc/src/__support/freestore.h | 376 +++++++++++++++++----
libc/src/__support/freetrie.h | 6 +-
libc/test/src/__support/freestore_test.cpp | 27 +-
libc/test/src/__support/freetrie_test.cpp | 16 +
6 files changed, 345 insertions(+), 84 deletions(-)
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index 48e70c7c29df6..ae2de684b3a24 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -41,6 +41,9 @@ class FreeList {
/// @returns The inner size of blocks in the list containing this node.
LIBC_INLINE size_t size() const { return block().inner_size(); }
+ /// @returns The next node in the list containing this node.
+ LIBC_INLINE Node *next_node() const { return next; }
+
private:
// Circularly linked pointers to adjacent nodes.
Node *prev;
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 73a80754050dd..d8d937eba50d8 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -85,7 +85,6 @@ LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
- free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
free_store.insert(block);
is_initialized = true;
}
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index adc0e061ace93..9394972f3adcc 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -7,113 +7,349 @@
//===----------------------------------------------------------------------===//
///
/// \file
-/// Interface for freestore.
+/// This file contains a two-level segregated fit free block store.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
#define LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
-#include "freetrie.h"
+#include "hdr/stdint_proxy.h"
+#include "hdr/types/size_t.h"
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/block.h"
+#include "src/__support/freelist.h"
+#include "src/__support/freetrie.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
namespace LIBC_NAMESPACE_DECL {
-/// A best-fit store of variously-sized free blocks. Blocks can be inserted and
-/// removed in logarithmic time.
-class FreeStore {
- friend class FreeListHeap;
-
-public:
- FreeStore() = default;
- FreeStore(const FreeStore &other) = delete;
- FreeStore &operator=(const FreeStore &other) = delete;
-
- /// Sets the range of possible block sizes. This can only be called when the
- /// trie is empty.
- LIBC_INLINE void set_range(FreeTrie::SizeRange range) {
- large_trie.set_range(range);
- }
-
- /// Insert a free block. If the block is too small to be tracked, nothing
- /// happens.
- void insert(BlockRef block);
+/// Configuration for TLSFFreeStore.
+template <size_t UNIT_SIZE_VAL, size_t STEP_SIZE_BITS_VAL,
+ size_t NUM_STEP_BITS_VAL, size_t NUM_TABLE_ENTRIES_VAL,
+ bool USE_TRIE_FOR_OVERFLOW_BIN_VAL = false>
+struct TLSFFreeStoreConfig {
+ static constexpr size_t UNIT_SIZE = UNIT_SIZE_VAL;
+ static constexpr size_t STEP_SIZE_BITS = STEP_SIZE_BITS_VAL;
+ static constexpr size_t NUM_STEP_BITS = NUM_STEP_BITS_VAL;
+ static constexpr size_t NUM_TABLE_ENTRIES = NUM_TABLE_ENTRIES_VAL;
+ static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN =
+ USE_TRIE_FOR_OVERFLOW_BIN_VAL;
+};
- /// Remove a free block. If the block is too small to be tracked, nothing
- /// happens.
- void remove(BlockRef block);
+// A two-level segregated fit store for free blocks.
+//
+// The store starts with small lists that grow linearly for small sizes, which
+// covers [0, ... UNIT_SIZE * EXP_BASE]. For larger sizes, the bits are managed
+// in a 2-D table. One can think of each row containing NUM_STEPS lists. Along
+// the row, the size grows by 2 exponentially; along the column, the size
+// increases by STEP_SIZE linearly.
+//
+// Mathematical layout:
+// STEP_SIZE = 1 << STEP_SIZE_BITS
+// NUM_STEPS = 1 << NUM_STEP_BITS
+// EXP_BASE = STEP_SIZE * NUM_STEPS
+// LARGE_SIZE_THRESHOLD = UNIT_SIZE * EXP_BASE
+//
+// Visual representation with example parameters:
+// UNIT_SIZE = 32, STEP_SIZE = 8, NUM_STEPS = 4
+// EXP_BASE = 32, THRESHOLD = 1024 B (1 KiB)
+//
+// 1. Small Sizes (Linear Array):
+// Covers [0, ... 1024 B] growing directly by UNIT_SIZE = 32 B
+// +-------+-------+-------+-------+-------+-----------+---------------+
+// | [0 B] | [32B] | [64B] | [96B] | ... | [992 B] | [1024 B (Th)] |
+// +-------+-------+-------+-------+-------+-----------+---------------+
+//
+// 2. Large Sizes (2-D Table):
+// Rows = FL (Exponential growth), Columns = SL (Linear steps)
+// One can think of each Row containing NUM_STEPS (4) lists.
+//
+// LINEAR INCREASE ALONG COLUMN (SL) --->
+// +---------------+---------------+---------------+---------------+
+// | Col = 0 | Col = 1 | Col = 2 | Col = 3 |
+// | (Base) | (+25% FL) | (+50% FL) | (+75% FL) |
+// +---------+---------------+---------------+---------------+---------------+
+// E | Row = 0 | 1024 B | 1280 B | 1536 B | 1792 B |
+// X |(Base 1K)| [1024 - 1279] | [1280 - 1535] | [1536 - 1791] | [1792 - 2047] |
+// P +---------+---------------+---------------+---------------+---------------+
+// O | Row = 1 | 2048 B | 2560 B | 3072 B | 3584 B |
+// N |(Base 2K)| [2048 - 2559] | [2560 - 3071] | [3072 - 3583] | [3584 - 4095] |
+// E +---------+---------------+---------------+---------------+---------------+
+// N | Row = 2 | 4096 B | 5120 B | 6144 B | 7168 B |
+// T |(Base 4K)| [4096 - 5119] | [5120 - 6143] | [6144 - 7167] | [7168 - 8191] |
+// I +---------+---------------+---------------+---------------+---------------+
+// A | Row = 3 | 8192 B | 10240 B | 12288 B | 14336 B |
+// L |(Base 8K)|[8192 - 10239]|[10240 - 12287]|[12288 - 14335]|[14336 - 16383]|
+// +---------+---------------+---------------+---------------+---------------+
+//
+// Note: For the real implementation, we don't actually store the lists in a
+// 2-D structure. Instead, we flatten the entire 2-D layout into a single
+// flat 1-D array of size TOTAL_BITS (free_lists), and map sizes directly to
+// a continuous 1-D index using size_to_bit_index. The allocation state is
+// tracked compactly in the lookup_table bitmask array.
+template <typename CONFIG> class TLSFFreeStoreImpl {
+protected:
+ static_assert(cpp::has_single_bit(CONFIG::UNIT_SIZE),
+ "unit size must be a power of two");
+ static_assert(CONFIG::NUM_TABLE_ENTRIES > 0,
+ "the lookup table must have at least one entry");
- /// Remove a best-fit free block that can contain the given size when
- /// allocated. Returns nullptr if there is no such block.
- BlockRef remove_best_fit(size_t size);
+ static constexpr size_t STEP_SIZE = size_t(1) << CONFIG::STEP_SIZE_BITS;
+ static constexpr size_t NUM_STEPS = size_t(1) << CONFIG::NUM_STEP_BITS;
+ static constexpr size_t EXP_BASE = STEP_SIZE * NUM_STEPS;
+ static constexpr int UNIT_SIZE_LOG2 = cpp::bit_width(CONFIG::UNIT_SIZE) - 1;
+ static constexpr int EXP_BASE_LOG2 =
+ CONFIG::STEP_SIZE_BITS + CONFIG::NUM_STEP_BITS;
+ static constexpr size_t BITS_PER_ENTRY =
+ cpp::numeric_limits<uintptr_t>::digits;
+ static constexpr size_t TOTAL_BITS =
+ CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
+ static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_OVERFLOW_BIN;
+ static constexpr size_t OVERFLOW_WIDTH =
+ size_t(1) << (cpp::numeric_limits<size_t>::digits - 2);
-private:
+public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
- static constexpr size_t MIN_LARGE_OUTER_SIZE = align_up(
- BlockRef::HEADER_SIZE + sizeof(FreeTrie::Node), BlockRef::MIN_ALIGN);
- static constexpr size_t NUM_SMALL_SIZES =
- (MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / BlockRef::MIN_ALIGN;
+ LIBC_INLINE TLSFFreeStoreImpl() = default;
+ LIBC_INLINE TLSFFreeStoreImpl(const TLSFFreeStoreImpl &other) = delete;
+ LIBC_INLINE TLSFFreeStoreImpl &
+ operator=(const TLSFFreeStoreImpl &other) = delete;
+
+ LIBC_INLINE void insert(BlockRef block);
+ LIBC_INLINE void remove(BlockRef block);
+ LIBC_INLINE BlockRef remove_best_fit(size_t size) {
+ return find_and_remove_fit(size);
+ }
+ LIBC_INLINE BlockRef find_and_remove_fit(size_t size);
+
+protected:
LIBC_INLINE static bool too_small(BlockRef block) {
return block.outer_size() < MIN_OUTER_SIZE;
}
- LIBC_INLINE static bool is_small(BlockRef block) {
- return block.outer_size() < MIN_LARGE_OUTER_SIZE;
- }
- FreeList &small_list(BlockRef block);
- FreeList *find_best_small_fit(size_t size);
+ union ListOrTrie {
+ FreeList list;
+ FreeTrie::Node *trie_root;
- cpp::array<FreeList, NUM_SMALL_SIZES> small_lists;
- FreeTrie large_trie;
+ LIBC_INLINE constexpr ListOrTrie() : trie_root(nullptr) {}
+ };
+
+ cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
+ cpp::array<ListOrTrie, TOTAL_BITS> free_lists{};
+
+ LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
+ LIBC_INLINE void set_bit(size_t bit_index);
+ LIBC_INLINE void clear_bit(size_t bit_index);
+ LIBC_INLINE bool get_bit(size_t bit_index) const;
+ LIBC_INLINE size_t find_first_bit_set_after(size_t bit_index) const;
+ LIBC_INLINE BlockRef remove_first_fit_in_list(size_t index, size_t size);
+ LIBC_INLINE FreeTrie get_trie();
+ LIBC_INLINE void set_trie(const FreeTrie &trie);
+ LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t size);
};
-LIBC_INLINE void FreeStore::insert(BlockRef block) {
+template <typename CONFIG>
+LIBC_INLINE constexpr size_t
+TLSFFreeStoreImpl<CONFIG>::size_to_bit_index(size_t size) {
+ if (size <= (EXP_BASE << UNIT_SIZE_LOG2))
+ return size >> UNIT_SIZE_LOG2;
+
+ size_t size_ilog2 = static_cast<size_t>(cpp::bit_width(size) - 1);
+ size_t exp_offset = (size_ilog2 - UNIT_SIZE_LOG2 - EXP_BASE_LOG2 - 1)
+ << CONFIG::NUM_STEP_BITS;
+ size_t step_index = size >> (size_ilog2 - CONFIG::NUM_STEP_BITS);
+ size_t index = EXP_BASE + exp_offset + step_index;
+
+ return index < TOTAL_BITS ? index : TOTAL_BITS - 1;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::set_bit(size_t bit_index) {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ lookup_table[entry_index] |= uintptr_t(1) << bit_offset;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::clear_bit(size_t bit_index) {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ lookup_table[entry_index] &= ~(uintptr_t(1) << bit_offset);
+}
+
+template <typename CONFIG>
+LIBC_INLINE bool TLSFFreeStoreImpl<CONFIG>::get_bit(size_t bit_index) const {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ return (lookup_table[entry_index] & (uintptr_t(1) << bit_offset)) != 0;
+}
+
+template <typename CONFIG>
+LIBC_INLINE size_t
+TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
+ if (bit_index >= TOTAL_BITS - 1)
+ return TOTAL_BITS;
+
+ size_t target_index = bit_index + 1;
+ size_t start_entry = target_index / BITS_PER_ENTRY;
+ size_t bit_offset = target_index % BITS_PER_ENTRY;
+
+ uintptr_t value = lookup_table[start_entry] & (~uintptr_t(0) << bit_offset);
+ if (value != 0)
+ return start_entry * BITS_PER_ENTRY +
+ static_cast<size_t>(cpp::countr_zero(value));
+
+ for (size_t i = start_entry + 1; i < CONFIG::NUM_TABLE_ENTRIES; ++i) {
+ value = lookup_table[i];
+ if (value != 0)
+ return i * BITS_PER_ENTRY + static_cast<size_t>(cpp::countr_zero(value));
+ }
+ return TOTAL_BITS;
+}
+
+template <typename CONFIG>
+LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
+ return FreeTrie(FreeTrie::SizeRange(0, OVERFLOW_WIDTH),
+ free_lists[TOTAL_BITS - 1].trie_root);
+}
+
+template <typename CONFIG>
+LIBC_INLINE void
+TLSFFreeStoreImpl<CONFIG>::set_trie(const FreeTrie &trie) {
+ free_lists[TOTAL_BITS - 1].trie_root = trie.get_root();
+}
+
+template <typename CONFIG>
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit_in_trie(size_t size) {
+ FreeTrie trie = get_trie();
+ if (FreeTrie::Node *best_fit = trie.find_best_fit(size)) {
+ BlockRef block = best_fit->block();
+ trie.remove(best_fit);
+ set_trie(trie);
+ if (trie.empty())
+ clear_bit(TOTAL_BITS - 1);
+ return block;
+ }
+ return BlockRef();
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
if (too_small(block))
return;
- if (is_small(block))
- small_list(block).push(block);
- else
- large_trie.push(block);
+ size_t bit_index = size_to_bit_index(block.inner_size());
+
+ if constexpr (USE_TRIE)
+ if (bit_index == TOTAL_BITS - 1) {
+ FreeTrie trie = get_trie();
+ trie.push(block);
+ set_trie(trie);
+ set_bit(bit_index);
+ return;
+ }
+
+ free_lists[bit_index].list.push(block);
+ set_bit(bit_index);
}
-LIBC_INLINE void FreeStore::remove(BlockRef block) {
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
if (too_small(block))
return;
- if (is_small(block)) {
- small_list(block).remove(
- reinterpret_cast<FreeList::Node *>(block.usable_space()));
- } else {
- large_trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
- }
+ size_t bit_index = size_to_bit_index(block.inner_size());
+
+ if constexpr (USE_TRIE)
+ if (bit_index == TOTAL_BITS - 1) {
+ FreeTrie trie = get_trie();
+ trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
+ set_trie(trie);
+ if (trie.empty())
+ clear_bit(bit_index);
+ return;
+ }
+
+ free_lists[bit_index].list.remove(
+ reinterpret_cast<FreeList::Node *>(block.usable_space()));
+ if (free_lists[bit_index].list.empty())
+ clear_bit(bit_index);
}
-LIBC_INLINE BlockRef FreeStore::remove_best_fit(size_t size) {
- if (FreeList *list = find_best_small_fit(size)) {
- BlockRef block = list->front();
- list->pop();
- return block;
+template <typename CONFIG>
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
+ FreeList::Node *begin_node = free_lists[index].list.begin();
+ if (begin_node == nullptr)
+ return BlockRef();
+
+ FreeList::Node *cur = begin_node;
+ do {
+ if (cur->size() >= size) {
+ free_lists[index].list.remove(cur);
+ if (free_lists[index].list.empty())
+ clear_bit(index);
+ return cur->block();
+ }
+ cur = cur->next_node();
+ } while (cur != begin_node);
+
+ return BlockRef();
+}
+
+template <typename CONFIG>
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
+ size_t bit_index = size_to_bit_index(size);
+
+ if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1)) {
+ if constexpr (USE_TRIE)
+ return find_and_remove_fit_in_trie(size);
+ else
+ return remove_first_fit_in_list(TOTAL_BITS - 1, size);
}
- if (FreeTrie::Node *best_fit = large_trie.find_best_fit(size)) {
- BlockRef block = best_fit->block();
- large_trie.remove(best_fit);
+
+ // 1. Try oversized bins (guaranteed fit, but larger).
+ size_t oversized_bit = find_first_bit_set_after(bit_index);
+ if (LIBC_LIKELY(oversized_bit < TOTAL_BITS)) {
+ if constexpr (USE_TRIE) {
+ if (oversized_bit == TOTAL_BITS - 1)
+ return find_and_remove_fit_in_trie(size);
+ }
+
+ BlockRef block = free_lists[oversized_bit].list.front();
+ free_lists[oversized_bit].list.pop();
+ if (free_lists[oversized_bit].list.empty())
+ clear_bit(oversized_bit);
return block;
}
+
+ // 2. Try exact fit (fallback).
+ if (get_bit(bit_index)) {
+ if (BlockRef block = remove_first_fit_in_list(bit_index, size))
+ return block;
+ }
+
return BlockRef();
}
-LIBC_INLINE FreeList &FreeStore::small_list(BlockRef block) {
- LIBC_ASSERT(is_small(block) && "only legal for small blocks");
- return small_lists[(block.outer_size() - MIN_OUTER_SIZE) /
- BlockRef::MIN_ALIGN];
-}
+template <size_t UNIT_SIZE, size_t STEP_SIZE_BITS, size_t NUM_STEP_BITS,
+ size_t NUM_TABLE_ENTRIES, bool USE_TRIE = false>
+using TLSFFreeStore = TLSFFreeStoreImpl<TLSFFreeStoreConfig<
+ UNIT_SIZE, STEP_SIZE_BITS, NUM_STEP_BITS, NUM_TABLE_ENTRIES, USE_TRIE>>;
-LIBC_INLINE FreeList *FreeStore::find_best_small_fit(size_t size) {
- for (FreeList &list : small_lists)
- if (!list.empty() && list.size() >= size)
- return &list;
- return nullptr;
-}
+#ifndef LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN
+#define LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN false
+#endif
+
+using FreeStore =
+ TLSFFreeStore<BlockRef::MIN_ALIGN, 3, 2, (sizeof(uintptr_t) == 8 ? 3 : 6),
+ LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN>;
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 9e35463462b38..728d1ad4f294f 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -88,7 +88,8 @@ class FreeTrie {
};
LIBC_INLINE constexpr FreeTrie() : FreeTrie(SizeRange{0, 0}) {}
- LIBC_INLINE constexpr FreeTrie(SizeRange range) : range(range) {}
+ LIBC_INLINE constexpr FreeTrie(SizeRange range, Node *root = nullptr)
+ : root(root), range(range) {}
/// Sets the range of possible block sizes. This can only be called when the
/// trie is empty.
@@ -100,6 +101,9 @@ class FreeTrie {
/// @returns Whether the trie contains any blocks.
LIBC_INLINE bool empty() const { return !root; }
+ /// @returns The root node of the trie.
+ LIBC_INLINE Node *get_root() const { return root; }
+
/// Push a block to the trie.
void push(BlockRef block);
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index 61103a9126c08..ce655367da00f 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -38,7 +38,6 @@ TEST(LlvmLibcFreeStore, TooSmall) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(too_small);
store.insert(remainder);
@@ -68,21 +67,26 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(smallest);
if (largest_small != smallest)
store.insert(largest_small);
store.insert(remainder);
- // Find exact match for smallest.
- ASSERT_EQ(store.remove_best_fit(smallest.inner_size()).addr(),
- smallest.addr());
- store.insert(smallest);
-
- // Find exact match for largest.
- ASSERT_EQ(store.remove_best_fit(largest_small.inner_size()).addr(),
- largest_small.addr());
- store.insert(largest_small);
+ // For TLSF (oversized first), asking for a size will return the block from
+ // the first non-empty oversized bin if one exists, bypassing the exact bin.
+ if (largest_small != smallest) {
+ BlockRef block = store.remove_best_fit(smallest.inner_size());
+ ASSERT_EQ(block.addr(), largest_small.addr());
+ store.insert(block);
+
+ BlockRef block2 = store.remove_best_fit(largest_small.inner_size());
+ ASSERT_EQ(block2.addr(), remainder.addr());
+ store.insert(block2);
+ } else {
+ BlockRef block = store.remove_best_fit(smallest.inner_size());
+ ASSERT_EQ(block.addr(), remainder.addr());
+ store.insert(block);
+ }
// Search small list for best fit.
BlockRef next_smallest =
@@ -108,7 +112,6 @@ TEST(LlvmLibcFreeStore, Remove) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(small);
store.insert(remainder);
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index bf3284b2faf64..17a81fa41e9a1 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -132,3 +132,19 @@ TEST(LlvmLibcFreeTrie, Remove) {
EXPECT_EQ(trie.find_best_fit(large.inner_size())->block().addr(),
large.addr());
}
+
+TEST(LlvmLibcFreeTrie, ConstructorWithRoot) {
+ FreeTrie::Node *root = nullptr;
+ FreeTrie trie({0, 4096}, root);
+ EXPECT_TRUE(trie.empty());
+ EXPECT_EQ(trie.get_root(), static_cast<FreeTrie::Node *>(nullptr));
+
+ byte mem[1024];
+ optional<BlockRef> maybeBlock = BlockRef::init(mem);
+ ASSERT_TRUE(maybeBlock.has_value());
+ BlockRef block = *maybeBlock;
+ trie.push(block);
+
+ EXPECT_FALSE(trie.empty());
+ EXPECT_NE(trie.get_root(), static_cast<FreeTrie::Node *>(nullptr));
+}
>From 2565f7f4075e62ffbdee69c8bfa59e56a36a9f3a Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:38:44 -0700
Subject: [PATCH 02/14] [libc] clean up extra changes
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freelist.cpp | 3 -
libc/src/__support/freelist_heap.h | 1 +
libc/src/__support/freestore.h | 36 ++++++++++--
libc/test/src/__support/freestore_test.cpp | 68 +++++++++++++---------
4 files changed, 74 insertions(+), 34 deletions(-)
diff --git a/libc/src/__support/freelist.cpp b/libc/src/__support/freelist.cpp
index 2fdcaadbdb554..2f468b08e4f7a 100644
--- a/libc/src/__support/freelist.cpp
+++ b/libc/src/__support/freelist.cpp
@@ -17,9 +17,6 @@ namespace LIBC_NAMESPACE_DECL {
void FreeList::push(Node *node) {
if (begin_) {
- LIBC_ASSERT(BlockRef::from_usable_space(node).outer_size() ==
- begin_->block().outer_size() &&
- "freelist entries must have the same size");
// Since the list is circular, insert the node immediately before begin_.
node->prev = begin_->prev;
node->next = begin_;
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index d8d937eba50d8..73a80754050dd 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -85,6 +85,7 @@ LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
+ free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
free_store.insert(block);
is_initialized = true;
}
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 9394972f3adcc..d72c9c6308c5a 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -109,8 +109,6 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
static constexpr size_t TOTAL_BITS =
CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_OVERFLOW_BIN;
- static constexpr size_t OVERFLOW_WIDTH =
- size_t(1) << (cpp::numeric_limits<size_t>::digits - 2);
public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
@@ -121,6 +119,8 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
LIBC_INLINE TLSFFreeStoreImpl &
operator=(const TLSFFreeStoreImpl &other) = delete;
+ LIBC_INLINE static constexpr size_t index_to_min_size(size_t index);
+ LIBC_INLINE void set_range(FreeTrie::SizeRange range);
LIBC_INLINE void insert(BlockRef block);
LIBC_INLINE void remove(BlockRef block);
LIBC_INLINE BlockRef remove_best_fit(size_t size) {
@@ -142,6 +142,7 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
cpp::array<ListOrTrie, TOTAL_BITS> free_lists{};
+ FreeTrie::SizeRange trie_range{index_to_min_size(TOTAL_BITS - 1), 1};
LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
LIBC_INLINE void set_bit(size_t bit_index);
@@ -213,10 +214,37 @@ TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
return TOTAL_BITS;
}
+template <typename CONFIG>
+LIBC_INLINE constexpr size_t
+TLSFFreeStoreImpl<CONFIG>::index_to_min_size(size_t index) {
+ if (index <= EXP_BASE)
+ return index << UNIT_SIZE_LOG2;
+
+ size_t local_index = index - EXP_BASE;
+ size_t exp_index = local_index >> CONFIG::NUM_STEP_BITS;
+ size_t linear_index = local_index & (NUM_STEPS - 1);
+
+ size_t row_base = (EXP_BASE << exp_index) << UNIT_SIZE_LOG2;
+ size_t step_size = (STEP_SIZE << exp_index) << UNIT_SIZE_LOG2;
+ return row_base + linear_index * step_size;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void
+TLSFFreeStoreImpl<CONFIG>::set_range(FreeTrie::SizeRange range) {
+ if constexpr (USE_TRIE) {
+ size_t heap_max = range.min + range.width;
+ size_t overflow_min = index_to_min_size(TOTAL_BITS - 1);
+ size_t width = 1;
+ if (heap_max > overflow_min)
+ width = cpp::bit_ceil(heap_max - overflow_min);
+ trie_range = FreeTrie::SizeRange(overflow_min, width);
+ }
+}
+
template <typename CONFIG>
LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
- return FreeTrie(FreeTrie::SizeRange(0, OVERFLOW_WIDTH),
- free_lists[TOTAL_BITS - 1].trie_root);
+ return FreeTrie(trie_range, free_lists[TOTAL_BITS - 1].trie_root);
}
template <typename CONFIG>
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index ce655367da00f..d56ed96143934 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -38,6 +38,7 @@ TEST(LlvmLibcFreeStore, TooSmall) {
BlockRef remainder = *maybeBlock;
FreeStore store;
+ store.set_range({0, 4096});
store.insert(too_small);
store.insert(remainder);
@@ -46,7 +47,7 @@ TEST(LlvmLibcFreeStore, TooSmall) {
store.remove(too_small);
}
-TEST(LlvmLibcFreeStore, RemoveBestFit) {
+TEST(LlvmLibcFreeStore, RemoveFit) {
byte mem[1024];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
@@ -67,37 +68,34 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) {
BlockRef remainder = *maybeBlock;
FreeStore store;
+ store.set_range({0, 4096});
store.insert(smallest);
if (largest_small != smallest)
store.insert(largest_small);
store.insert(remainder);
- // For TLSF (oversized first), asking for a size will return the block from
- // the first non-empty oversized bin if one exists, bypassing the exact bin.
- if (largest_small != smallest) {
- BlockRef block = store.remove_best_fit(smallest.inner_size());
- ASSERT_EQ(block.addr(), largest_small.addr());
- store.insert(block);
-
- BlockRef block2 = store.remove_best_fit(largest_small.inner_size());
- ASSERT_EQ(block2.addr(), remainder.addr());
- store.insert(block2);
- } else {
- BlockRef block = store.remove_best_fit(smallest.inner_size());
- ASSERT_EQ(block.addr(), remainder.addr());
- store.insert(block);
- }
-
- // Search small list for best fit.
- BlockRef next_smallest =
- largest_small == smallest ? remainder : largest_small;
- ASSERT_EQ(store.remove_best_fit(smallest.inner_size() + 1).addr(),
- next_smallest.addr());
- store.insert(next_smallest);
-
- // Continue search for best fit to large blocks.
- EXPECT_EQ(store.remove_best_fit(largest_small.inner_size() + 1).addr(),
- remainder.addr());
+ // Requesting smallest size returns a valid block fitting the size.
+ BlockRef block1 = store.remove_best_fit(smallest.inner_size());
+ ASSERT_NE(block1.addr(), BlockRef().addr());
+ ASSERT_GE(block1.inner_size(), smallest.inner_size());
+ store.insert(block1);
+
+ // Requesting largest_small size returns a valid block fitting the size.
+ BlockRef block2 = store.remove_best_fit(largest_small.inner_size());
+ ASSERT_NE(block2.addr(), BlockRef().addr());
+ ASSERT_GE(block2.inner_size(), largest_small.inner_size());
+ store.insert(block2);
+
+ // Requesting smallest inner_size + 1 returns a valid block fitting the size.
+ BlockRef block3 = store.remove_best_fit(smallest.inner_size() + 1);
+ ASSERT_NE(block3.addr(), BlockRef().addr());
+ ASSERT_GE(block3.inner_size(), smallest.inner_size() + 1);
+ store.insert(block3);
+
+ // Requesting largest_small inner_size + 1 returns a valid block fitting the size.
+ BlockRef block4 = store.remove_best_fit(largest_small.inner_size() + 1);
+ ASSERT_NE(block4.addr(), BlockRef().addr());
+ ASSERT_GE(block4.inner_size(), largest_small.inner_size() + 1);
}
TEST(LlvmLibcFreeStore, Remove) {
@@ -112,6 +110,7 @@ TEST(LlvmLibcFreeStore, Remove) {
BlockRef remainder = *maybeBlock;
FreeStore store;
+ store.set_range({0, 4096});
store.insert(small);
store.insert(remainder);
@@ -122,3 +121,18 @@ TEST(LlvmLibcFreeStore, Remove) {
ASSERT_EQ(store.remove_best_fit(small.inner_size()).addr(),
BlockRef().addr());
}
+
+TEST(LlvmLibcFreeStore, IndexToMinSize) {
+ constexpr size_t min_size_0 = FreeStore::index_to_min_size(0);
+ EXPECT_EQ(min_size_0, static_cast<size_t>(0));
+
+ constexpr size_t min_size_1 = FreeStore::index_to_min_size(1);
+ EXPECT_EQ(min_size_1, static_cast<size_t>(BlockRef::MIN_ALIGN));
+
+ size_t prev_size = 0;
+ for (size_t i = 1; i < 64; ++i) {
+ size_t min_size = FreeStore::index_to_min_size(i);
+ EXPECT_GT(min_size, prev_size);
+ prev_size = min_size;
+ }
+}
>From 48c586d949b22465e614311d812ac413b7d74698 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:41:00 -0700
Subject: [PATCH 03/14] [libc] rename next_node to next and get_root to root
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freelist.cpp | 18 +++++++++---------
libc/src/__support/freelist.h | 6 +++---
libc/src/__support/freestore.h | 4 ++--
libc/src/__support/freetrie.cpp | 4 ++--
libc/src/__support/freetrie.h | 14 +++++++-------
libc/test/src/__support/freetrie_test.cpp | 4 ++--
6 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/libc/src/__support/freelist.cpp b/libc/src/__support/freelist.cpp
index 2f468b08e4f7a..c0834fe6029d8 100644
--- a/libc/src/__support/freelist.cpp
+++ b/libc/src/__support/freelist.cpp
@@ -18,26 +18,26 @@ namespace LIBC_NAMESPACE_DECL {
void FreeList::push(Node *node) {
if (begin_) {
// Since the list is circular, insert the node immediately before begin_.
- node->prev = begin_->prev;
- node->next = begin_;
- begin_->prev->next = node;
- begin_->prev = node;
+ node->prev_ = begin_->prev_;
+ node->next_ = begin_;
+ begin_->prev_->next_ = node;
+ begin_->prev_ = node;
} else {
- begin_ = node->prev = node->next = node;
+ begin_ = node->prev_ = node->next_ = node;
}
}
void FreeList::remove(Node *node) {
LIBC_ASSERT(begin_ && "cannot remove from empty list");
- Node *next = node->next;
+ Node *next = node->next_;
if (node == next) {
LIBC_ASSERT(node == begin_ &&
"a self-referential node must be the only element");
begin_ = nullptr;
} else {
- Node *prev = node->prev;
- prev->next = next;
- next->prev = prev;
+ Node *prev = node->prev_;
+ prev->next_ = next;
+ next->prev_ = prev;
if (begin_ == node)
begin_ = next;
}
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index ae2de684b3a24..4bac60336761f 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -42,12 +42,12 @@ class FreeList {
LIBC_INLINE size_t size() const { return block().inner_size(); }
/// @returns The next node in the list containing this node.
- LIBC_INLINE Node *next_node() const { return next; }
+ LIBC_INLINE Node *next() const { return next_; }
private:
// Circularly linked pointers to adjacent nodes.
- Node *prev;
- Node *next;
+ Node *prev_;
+ Node *next_;
friend class FreeList;
};
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index d72c9c6308c5a..6f832100bd1d4 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -250,7 +250,7 @@ LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
template <typename CONFIG>
LIBC_INLINE void
TLSFFreeStoreImpl<CONFIG>::set_trie(const FreeTrie &trie) {
- free_lists[TOTAL_BITS - 1].trie_root = trie.get_root();
+ free_lists[TOTAL_BITS - 1].trie_root = trie.root();
}
template <typename CONFIG>
@@ -324,7 +324,7 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
clear_bit(index);
return cur->block();
}
- cur = cur->next_node();
+ cur = cur->next();
} while (cur != begin_node);
return BlockRef();
diff --git a/libc/src/__support/freetrie.cpp b/libc/src/__support/freetrie.cpp
index e76efe717f215..e3342080b623a 100644
--- a/libc/src/__support/freetrie.cpp
+++ b/libc/src/__support/freetrie.cpp
@@ -52,8 +52,8 @@ void FreeTrie::replace_node(Node *node, Node *new_node) {
"no reference to child node found in parent");
parent_child = new_node;
} else {
- LIBC_ASSERT(root == node && "non-root node had no parent");
- root = new_node;
+ LIBC_ASSERT(root_ == node && "non-root node had no parent");
+ root_ = new_node;
}
if (node->lower)
node->lower->parent = new_node;
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 728d1ad4f294f..9e0d61b350065 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -89,7 +89,7 @@ class FreeTrie {
LIBC_INLINE constexpr FreeTrie() : FreeTrie(SizeRange{0, 0}) {}
LIBC_INLINE constexpr FreeTrie(SizeRange range, Node *root = nullptr)
- : root(root), range(range) {}
+ : root_(root), range(range) {}
/// Sets the range of possible block sizes. This can only be called when the
/// trie is empty.
@@ -99,10 +99,10 @@ class FreeTrie {
}
/// @returns Whether the trie contains any blocks.
- LIBC_INLINE bool empty() const { return !root; }
+ LIBC_INLINE bool empty() const { return !root_; }
/// @returns The root node of the trie.
- LIBC_INLINE Node *get_root() const { return root; }
+ LIBC_INLINE Node *root() const { return root_; }
/// Push a block to the trie.
void push(BlockRef block);
@@ -116,13 +116,13 @@ class FreeTrie {
private:
/// @returns Whether a node is the head of its containing freelist.
- bool is_head(Node *node) const { return node->parent || node == root; }
+ bool is_head(Node *node) const { return node->parent || node == root_; }
/// Replaces references to one node with another (or nullptr) in all adjacent
/// parent and child nodes.
void replace_node(Node *node, Node *new_node);
- Node *root = nullptr;
+ Node *root_ = nullptr;
SizeRange range;
};
@@ -133,7 +133,7 @@ LIBC_INLINE void FreeTrie::push(BlockRef block) {
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
// Find the position in the tree to push to.
- Node **cur = &root;
+ Node **cur = &root_;
Node *parent = nullptr;
SizeRange cur_range = range;
while (*cur && (*cur)->size() != size) {
@@ -164,7 +164,7 @@ LIBC_INLINE FreeTrie::Node *FreeTrie::find_best_fit(size_t size) {
if (empty() || range.max() < size)
return nullptr;
- Node *cur = root;
+ Node *cur = root_;
SizeRange cur_range = range;
Node *best_fit = nullptr;
Node *deferred_upper_trie = nullptr;
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 17a81fa41e9a1..267c6eb7d8894 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -137,7 +137,7 @@ TEST(LlvmLibcFreeTrie, ConstructorWithRoot) {
FreeTrie::Node *root = nullptr;
FreeTrie trie({0, 4096}, root);
EXPECT_TRUE(trie.empty());
- EXPECT_EQ(trie.get_root(), static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_EQ(trie.root(), static_cast<FreeTrie::Node *>(nullptr));
byte mem[1024];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
@@ -146,5 +146,5 @@ TEST(LlvmLibcFreeTrie, ConstructorWithRoot) {
trie.push(block);
EXPECT_FALSE(trie.empty());
- EXPECT_NE(trie.get_root(), static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_NE(trie.root(), static_cast<FreeTrie::Node *>(nullptr));
}
>From 6a2662c18ea298e824790cc331e31fa06da4389a Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:45:54 -0700
Subject: [PATCH 04/14] [libc] refactor DefaultFreeStoreConfig for
TLSFFreeStore
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freestore.h | 32 ++++++++++----------------------
1 file changed, 10 insertions(+), 22 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 6f832100bd1d4..4ccf57c505595 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -27,17 +27,13 @@
namespace LIBC_NAMESPACE_DECL {
-/// Configuration for TLSFFreeStore.
-template <size_t UNIT_SIZE_VAL, size_t STEP_SIZE_BITS_VAL,
- size_t NUM_STEP_BITS_VAL, size_t NUM_TABLE_ENTRIES_VAL,
- bool USE_TRIE_FOR_OVERFLOW_BIN_VAL = false>
-struct TLSFFreeStoreConfig {
- static constexpr size_t UNIT_SIZE = UNIT_SIZE_VAL;
- static constexpr size_t STEP_SIZE_BITS = STEP_SIZE_BITS_VAL;
- static constexpr size_t NUM_STEP_BITS = NUM_STEP_BITS_VAL;
- static constexpr size_t NUM_TABLE_ENTRIES = NUM_TABLE_ENTRIES_VAL;
- static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN =
- USE_TRIE_FOR_OVERFLOW_BIN_VAL;
+/// Default configuration for TLSFFreeStore.
+struct DefaultFreeStoreConfig {
+ static constexpr size_t UNIT_SIZE = BlockRef::MIN_ALIGN;
+ static constexpr size_t STEP_SIZE_BITS = 3;
+ static constexpr size_t NUM_STEP_BITS = 2;
+ static constexpr size_t NUM_TABLE_ENTRIES = sizeof(uintptr_t) == 8 ? 3 : 6;
+ static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN = true;
};
// A two-level segregated fit store for free blocks.
@@ -366,18 +362,10 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
return BlockRef();
}
-template <size_t UNIT_SIZE, size_t STEP_SIZE_BITS, size_t NUM_STEP_BITS,
- size_t NUM_TABLE_ENTRIES, bool USE_TRIE = false>
-using TLSFFreeStore = TLSFFreeStoreImpl<TLSFFreeStoreConfig<
- UNIT_SIZE, STEP_SIZE_BITS, NUM_STEP_BITS, NUM_TABLE_ENTRIES, USE_TRIE>>;
+template <typename CONFIG = DefaultFreeStoreConfig>
+using TLSFFreeStore = TLSFFreeStoreImpl<CONFIG>;
-#ifndef LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN
-#define LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN false
-#endif
-
-using FreeStore =
- TLSFFreeStore<BlockRef::MIN_ALIGN, 3, 2, (sizeof(uintptr_t) == 8 ? 3 : 6),
- LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN>;
+using FreeStore = TLSFFreeStore<DefaultFreeStoreConfig>;
} // namespace LIBC_NAMESPACE_DECL
>From b50021a5a6da66105d0cc8d40f2008550e385cf7 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:47:43 -0700
Subject: [PATCH 05/14] [libc] introduce LINEAR_SCAN_LIMIT to FreeStore config
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freestore.h | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 4ccf57c505595..721abb244985a 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -34,6 +34,7 @@ struct DefaultFreeStoreConfig {
static constexpr size_t NUM_STEP_BITS = 2;
static constexpr size_t NUM_TABLE_ENTRIES = sizeof(uintptr_t) == 8 ? 3 : 6;
static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN = true;
+ static constexpr size_t LINEAR_SCAN_LIMIT = 16;
};
// A two-level segregated fit store for free blocks.
@@ -313,6 +314,7 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
return BlockRef();
FreeList::Node *cur = begin_node;
+ size_t count = 0;
do {
if (cur->size() >= size) {
free_lists[index].list.remove(cur);
@@ -321,7 +323,8 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
return cur->block();
}
cur = cur->next();
- } while (cur != begin_node);
+ ++count;
+ } while (cur != begin_node && count < CONFIG::LINEAR_SCAN_LIMIT);
return BlockRef();
}
>From ad665d6a42db610da3d9a06e78da638085fb08d5 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:50:48 -0700
Subject: [PATCH 06/14] [libc] replace union ListOrTrie with struct using
uintptr_t payload and bit_cast
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freestore.h | 48 +++++++++++++++++++++++-----------
1 file changed, 33 insertions(+), 15 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 721abb244985a..e4ac5d193d7c3 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -130,11 +130,21 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
return block.outer_size() < MIN_OUTER_SIZE;
}
- union ListOrTrie {
- FreeList list;
- FreeTrie::Node *trie_root;
+ struct ListOrTrie {
+ uintptr_t payload = 0;
- LIBC_INLINE constexpr ListOrTrie() : trie_root(nullptr) {}
+ LIBC_INLINE FreeList list() const {
+ return FreeList(cpp::bit_cast<FreeList::Node *>(payload));
+ }
+ LIBC_INLINE void set_list(FreeList l) {
+ payload = cpp::bit_cast<uintptr_t>(l.begin());
+ }
+ LIBC_INLINE FreeTrie::Node *trie_root() const {
+ return cpp::bit_cast<FreeTrie::Node *>(payload);
+ }
+ LIBC_INLINE void set_trie_root(FreeTrie::Node *r) {
+ payload = cpp::bit_cast<uintptr_t>(r);
+ }
};
cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
@@ -241,13 +251,13 @@ TLSFFreeStoreImpl<CONFIG>::set_range(FreeTrie::SizeRange range) {
template <typename CONFIG>
LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
- return FreeTrie(trie_range, free_lists[TOTAL_BITS - 1].trie_root);
+ return FreeTrie(trie_range, free_lists[TOTAL_BITS - 1].trie_root());
}
template <typename CONFIG>
LIBC_INLINE void
TLSFFreeStoreImpl<CONFIG>::set_trie(const FreeTrie &trie) {
- free_lists[TOTAL_BITS - 1].trie_root = trie.root();
+ free_lists[TOTAL_BITS - 1].set_trie_root(trie.root());
}
template <typename CONFIG>
@@ -280,7 +290,9 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
return;
}
- free_lists[bit_index].list.push(block);
+ FreeList list = free_lists[bit_index].list();
+ list.push(block);
+ free_lists[bit_index].set_list(list);
set_bit(bit_index);
}
@@ -300,16 +312,19 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
return;
}
- free_lists[bit_index].list.remove(
+ FreeList list = free_lists[bit_index].list();
+ list.remove(
reinterpret_cast<FreeList::Node *>(block.usable_space()));
- if (free_lists[bit_index].list.empty())
+ if (list.empty())
clear_bit(bit_index);
+ free_lists[bit_index].set_list(list);
}
template <typename CONFIG>
LIBC_INLINE BlockRef
TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
- FreeList::Node *begin_node = free_lists[index].list.begin();
+ FreeList list = free_lists[index].list();
+ FreeList::Node *begin_node = list.begin();
if (begin_node == nullptr)
return BlockRef();
@@ -317,9 +332,10 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
size_t count = 0;
do {
if (cur->size() >= size) {
- free_lists[index].list.remove(cur);
- if (free_lists[index].list.empty())
+ list.remove(cur);
+ if (list.empty())
clear_bit(index);
+ free_lists[index].set_list(list);
return cur->block();
}
cur = cur->next();
@@ -349,10 +365,12 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
return find_and_remove_fit_in_trie(size);
}
- BlockRef block = free_lists[oversized_bit].list.front();
- free_lists[oversized_bit].list.pop();
- if (free_lists[oversized_bit].list.empty())
+ FreeList list = free_lists[oversized_bit].list();
+ BlockRef block = list.front();
+ list.pop();
+ if (list.empty())
clear_bit(oversized_bit);
+ free_lists[oversized_bit].set_list(list);
return block;
}
>From 2d20d7392609d0869ebf5578a25713c49fd9826a Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:52:17 -0700
Subject: [PATCH 07/14] [libc] simplify TLSFFreeStoreImpl with
free_lists[BITS-1] and FreeTrie trie members
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freestore.h | 95 ++++++++++++----------------------
1 file changed, 32 insertions(+), 63 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index e4ac5d193d7c3..489218c584ab3 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -130,26 +130,10 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
return block.outer_size() < MIN_OUTER_SIZE;
}
- struct ListOrTrie {
- uintptr_t payload = 0;
-
- LIBC_INLINE FreeList list() const {
- return FreeList(cpp::bit_cast<FreeList::Node *>(payload));
- }
- LIBC_INLINE void set_list(FreeList l) {
- payload = cpp::bit_cast<uintptr_t>(l.begin());
- }
- LIBC_INLINE FreeTrie::Node *trie_root() const {
- return cpp::bit_cast<FreeTrie::Node *>(payload);
- }
- LIBC_INLINE void set_trie_root(FreeTrie::Node *r) {
- payload = cpp::bit_cast<uintptr_t>(r);
- }
- };
-
cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
- cpp::array<ListOrTrie, TOTAL_BITS> free_lists{};
- FreeTrie::SizeRange trie_range{index_to_min_size(TOTAL_BITS - 1), 1};
+ cpp::array<FreeList, TOTAL_BITS - 1> free_lists{};
+ FreeTrie trie{};
+ FreeList overflow_list{};
LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
LIBC_INLINE void set_bit(size_t bit_index);
@@ -157,8 +141,6 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
LIBC_INLINE bool get_bit(size_t bit_index) const;
LIBC_INLINE size_t find_first_bit_set_after(size_t bit_index) const;
LIBC_INLINE BlockRef remove_first_fit_in_list(size_t index, size_t size);
- LIBC_INLINE FreeTrie get_trie();
- LIBC_INLINE void set_trie(const FreeTrie &trie);
LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t size);
};
@@ -245,29 +227,16 @@ TLSFFreeStoreImpl<CONFIG>::set_range(FreeTrie::SizeRange range) {
size_t width = 1;
if (heap_max > overflow_min)
width = cpp::bit_ceil(heap_max - overflow_min);
- trie_range = FreeTrie::SizeRange(overflow_min, width);
+ trie.set_range(FreeTrie::SizeRange(overflow_min, width));
}
}
-template <typename CONFIG>
-LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
- return FreeTrie(trie_range, free_lists[TOTAL_BITS - 1].trie_root());
-}
-
-template <typename CONFIG>
-LIBC_INLINE void
-TLSFFreeStoreImpl<CONFIG>::set_trie(const FreeTrie &trie) {
- free_lists[TOTAL_BITS - 1].set_trie_root(trie.root());
-}
-
template <typename CONFIG>
LIBC_INLINE BlockRef
TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit_in_trie(size_t size) {
- FreeTrie trie = get_trie();
if (FreeTrie::Node *best_fit = trie.find_best_fit(size)) {
BlockRef block = best_fit->block();
trie.remove(best_fit);
- set_trie(trie);
if (trie.empty())
clear_bit(TOTAL_BITS - 1);
return block;
@@ -281,18 +250,16 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
return;
size_t bit_index = size_to_bit_index(block.inner_size());
- if constexpr (USE_TRIE)
- if (bit_index == TOTAL_BITS - 1) {
- FreeTrie trie = get_trie();
+ if (bit_index == TOTAL_BITS - 1) {
+ if constexpr (USE_TRIE)
trie.push(block);
- set_trie(trie);
- set_bit(bit_index);
- return;
- }
+ else
+ overflow_list.push(block);
+ set_bit(bit_index);
+ return;
+ }
- FreeList list = free_lists[bit_index].list();
- list.push(block);
- free_lists[bit_index].set_list(list);
+ free_lists[bit_index].push(block);
set_bit(bit_index);
}
@@ -302,28 +269,31 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
return;
size_t bit_index = size_to_bit_index(block.inner_size());
- if constexpr (USE_TRIE)
- if (bit_index == TOTAL_BITS - 1) {
- FreeTrie trie = get_trie();
+ if (bit_index == TOTAL_BITS - 1) {
+ if constexpr (USE_TRIE) {
trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
- set_trie(trie);
if (trie.empty())
clear_bit(bit_index);
- return;
+ } else {
+ overflow_list.remove(
+ reinterpret_cast<FreeList::Node *>(block.usable_space()));
+ if (overflow_list.empty())
+ clear_bit(bit_index);
}
+ return;
+ }
- FreeList list = free_lists[bit_index].list();
- list.remove(
+ free_lists[bit_index].remove(
reinterpret_cast<FreeList::Node *>(block.usable_space()));
- if (list.empty())
+ if (free_lists[bit_index].empty())
clear_bit(bit_index);
- free_lists[bit_index].set_list(list);
}
template <typename CONFIG>
LIBC_INLINE BlockRef
TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
- FreeList list = free_lists[index].list();
+ FreeList &list =
+ (index == TOTAL_BITS - 1) ? overflow_list : free_lists[index];
FreeList::Node *begin_node = list.begin();
if (begin_node == nullptr)
return BlockRef();
@@ -335,7 +305,6 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
list.remove(cur);
if (list.empty())
clear_bit(index);
- free_lists[index].set_list(list);
return cur->block();
}
cur = cur->next();
@@ -360,17 +329,17 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
// 1. Try oversized bins (guaranteed fit, but larger).
size_t oversized_bit = find_first_bit_set_after(bit_index);
if (LIBC_LIKELY(oversized_bit < TOTAL_BITS)) {
- if constexpr (USE_TRIE) {
- if (oversized_bit == TOTAL_BITS - 1)
+ if (oversized_bit == TOTAL_BITS - 1) {
+ if constexpr (USE_TRIE)
return find_and_remove_fit_in_trie(size);
+ else
+ return remove_first_fit_in_list(TOTAL_BITS - 1, size);
}
- FreeList list = free_lists[oversized_bit].list();
- BlockRef block = list.front();
- list.pop();
- if (list.empty())
+ BlockRef block = free_lists[oversized_bit].front();
+ free_lists[oversized_bit].pop();
+ if (free_lists[oversized_bit].empty())
clear_bit(oversized_bit);
- free_lists[oversized_bit].set_list(list);
return block;
}
>From d40a766df868b3bfac903827ed322ba4f4e0a322 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 22 Jul 2026 13:59:00 -0700
Subject: [PATCH 08/14] [libc] add fastpath if linear portion are exact
TAG=agy
CONV=f78b1e3e-dbac-4be5-bc9f-d37a6264139e
---
libc/src/__support/freestore.h | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 489218c584ab3..82a1f5f2df07e 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -317,6 +317,19 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
template <typename CONFIG>
LIBC_INLINE BlockRef
TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
+ if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
+ size_t index = align_up(size, CONFIG::UNIT_SIZE) >> UNIT_SIZE_LOG2;
+ if (LIBC_LIKELY(index <= EXP_BASE)) {
+ if (get_bit(index)) {
+ BlockRef block = free_lists[index].front();
+ free_lists[index].pop();
+ if (free_lists[index].empty())
+ clear_bit(index);
+ return block;
+ }
+ }
+ }
+
size_t bit_index = size_to_bit_index(size);
if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1)) {
>From e1871dcedc18738f23433422410dc360cef1f8d9 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Fri, 31 Jul 2026 13:37:40 -0700
Subject: [PATCH 09/14] [libc][__support] Implement exact linear binning for
TLSFFreeStoreImpl
Previously, size_to_bit_index used size >> UNIT_SIZE_LOG2 for linear bins,
which mapped size 24 to bin 1 and size 17 to bin 1, causing a mismatch
between allocation request sizes and physical block bucket sizes.
This change introduces exact mapping for linear bins:
- Introduces LINEAR_BINS to compute the exact number of linear bins needed
to reach the exponential table boundary (29 on MSVC Windows where UNIT_SIZE is 8,
31 on Linux where UNIT_SIZE is 16).
- Sizes <= MIN_INNER_SIZE map to bin 0.
- Larger linear sizes map to ((size - MIN_INNER_SIZE - 1) >> UNIT_SIZE_LOG2) + 1.
- Uses LINEAR_BINS as the base index for exponential bins in size_to_bit_index,
index_to_min_size, and find_and_remove_fit, ensuring strict monotonicity
across all indices without runtime clamping.
- Adds NegativeTestForFullHeap unit test in freelist_heap_test.cpp and updates
freestore_test.cpp.
TAG=agy
CONV=cff84e8c-ee22-4f39-af3c-344d1e6f417b
---
libc/src/__support/freestore.h | 53 +++++++++++++------
.../test/src/__support/freelist_heap_test.cpp | 9 ++++
libc/test/src/__support/freestore_test.cpp | 5 +-
3 files changed, 48 insertions(+), 19 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 82a1f5f2df07e..543caf3f9ca31 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -110,6 +110,14 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
+ static constexpr size_t MIN_INNER_SIZE =
+ MIN_OUTER_SIZE - BlockRef::HEADER_SIZE + BlockRef::PREV_FIELD_SIZE;
+ static constexpr size_t LINEAR_BINS =
+ (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN)
+ ? ((EXP_BASE << UNIT_SIZE_LOG2) - MIN_INNER_SIZE) /
+ CONFIG::UNIT_SIZE +
+ 1
+ : EXP_BASE;
LIBC_INLINE TLSFFreeStoreImpl() = default;
LIBC_INLINE TLSFFreeStoreImpl(const TLSFFreeStoreImpl &other) = delete;
@@ -124,6 +132,7 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
return find_and_remove_fit(size);
}
LIBC_INLINE BlockRef find_and_remove_fit(size_t size);
+ LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
protected:
LIBC_INLINE static bool too_small(BlockRef block) {
@@ -135,7 +144,6 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
FreeTrie trie{};
FreeList overflow_list{};
- LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
LIBC_INLINE void set_bit(size_t bit_index);
LIBC_INLINE void clear_bit(size_t bit_index);
LIBC_INLINE bool get_bit(size_t bit_index) const;
@@ -147,14 +155,20 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
template <typename CONFIG>
LIBC_INLINE constexpr size_t
TLSFFreeStoreImpl<CONFIG>::size_to_bit_index(size_t size) {
- if (size <= (EXP_BASE << UNIT_SIZE_LOG2))
+ if (size <= (EXP_BASE << UNIT_SIZE_LOG2)) {
+ if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
+ if (size <= MIN_INNER_SIZE)
+ return 0;
+ return ((size - MIN_INNER_SIZE - 1) >> UNIT_SIZE_LOG2) + 1;
+ }
return size >> UNIT_SIZE_LOG2;
+ }
size_t size_ilog2 = static_cast<size_t>(cpp::bit_width(size) - 1);
size_t exp_offset = (size_ilog2 - UNIT_SIZE_LOG2 - EXP_BASE_LOG2 - 1)
<< CONFIG::NUM_STEP_BITS;
size_t step_index = size >> (size_ilog2 - CONFIG::NUM_STEP_BITS);
- size_t index = EXP_BASE + exp_offset + step_index;
+ size_t index = LINEAR_BINS + exp_offset + step_index;
return index < TOTAL_BITS ? index : TOTAL_BITS - 1;
}
@@ -206,10 +220,16 @@ TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
template <typename CONFIG>
LIBC_INLINE constexpr size_t
TLSFFreeStoreImpl<CONFIG>::index_to_min_size(size_t index) {
- if (index <= EXP_BASE)
+ if (index < LINEAR_BINS) {
+ if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
+ if (index == 0)
+ return 0;
+ return MIN_INNER_SIZE + (index - 1) * CONFIG::UNIT_SIZE + 1;
+ }
return index << UNIT_SIZE_LOG2;
+ }
- size_t local_index = index - EXP_BASE;
+ size_t local_index = index - LINEAR_BINS;
size_t exp_index = local_index >> CONFIG::NUM_STEP_BITS;
size_t linear_index = local_index & (NUM_STEPS - 1);
@@ -317,21 +337,20 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
template <typename CONFIG>
LIBC_INLINE BlockRef
TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
+
+ size_t bit_index = size_to_bit_index(size);
+
+ // Fast path for small linear bins if UNIT_SIZE == MIN_ALIGN
if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
- size_t index = align_up(size, CONFIG::UNIT_SIZE) >> UNIT_SIZE_LOG2;
- if (LIBC_LIKELY(index <= EXP_BASE)) {
- if (get_bit(index)) {
- BlockRef block = free_lists[index].front();
- free_lists[index].pop();
- if (free_lists[index].empty())
- clear_bit(index);
- return block;
- }
+ if (LIBC_LIKELY(bit_index < LINEAR_BINS && get_bit(bit_index))) {
+ BlockRef block = free_lists[bit_index].front();
+ free_lists[bit_index].pop();
+ if (free_lists[bit_index].empty())
+ clear_bit(bit_index);
+ return block;
}
}
- size_t bit_index = size_to_bit_index(size);
-
if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1)) {
if constexpr (USE_TRIE)
return find_and_remove_fit_in_trie(size);
@@ -342,7 +361,7 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
// 1. Try oversized bins (guaranteed fit, but larger).
size_t oversized_bit = find_first_bit_set_after(bit_index);
if (LIBC_LIKELY(oversized_bit < TOTAL_BITS)) {
- if (oversized_bit == TOTAL_BITS - 1) {
+ if (LIBC_UNLIKELY(oversized_bit == TOTAL_BITS - 1)) {
if constexpr (USE_TRIE)
return find_and_remove_fit_in_trie(size);
else
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 1ee6bf0ce4ab4..4f5cdf32cd5f2 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -366,3 +366,12 @@ TEST_FOR_EACH_ALLOCATOR(AllocationSize, 2048) {
allocator.free(ptr);
EXPECT_EQ(allocator.allocation_size(ptr), size_t(0));
}
+
+TEST_FOR_EACH_ALLOCATOR(NegativeTestForFullHeap, 2048) {
+ // Numbers are selected to stress the first exponential bin for 64bit target.
+ void *ptr1 = allocator.allocate(528);
+ allocator.allocate(1000);
+ allocator.free(ptr1);
+ void *ptr3 = allocator.allocate(605);
+ EXPECT_EQ(ptr3, static_cast<void *>(nullptr));
+}
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index d56ed96143934..e4ee44d2776eb 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -92,7 +92,8 @@ TEST(LlvmLibcFreeStore, RemoveFit) {
ASSERT_GE(block3.inner_size(), smallest.inner_size() + 1);
store.insert(block3);
- // Requesting largest_small inner_size + 1 returns a valid block fitting the size.
+ // Requesting largest_small inner_size + 1 returns a valid block fitting the
+ // size.
BlockRef block4 = store.remove_best_fit(largest_small.inner_size() + 1);
ASSERT_NE(block4.addr(), BlockRef().addr());
ASSERT_GE(block4.inner_size(), largest_small.inner_size() + 1);
@@ -127,7 +128,7 @@ TEST(LlvmLibcFreeStore, IndexToMinSize) {
EXPECT_EQ(min_size_0, static_cast<size_t>(0));
constexpr size_t min_size_1 = FreeStore::index_to_min_size(1);
- EXPECT_EQ(min_size_1, static_cast<size_t>(BlockRef::MIN_ALIGN));
+ EXPECT_EQ(min_size_1, static_cast<size_t>(FreeStore::MIN_INNER_SIZE + 1));
size_t prev_size = 0;
for (size_t i = 1; i < 64; ++i) {
>From 58719932bd123e6319feaaf49f36f36a8f6008fd Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Mon, 3 Aug 2026 11:30:22 -0700
Subject: [PATCH 10/14] address CR: simplify and comment the bin computation
logic
TAG=agy
CONV=e64ff65b-c845-4136-9173-da6f615197ee
---
libc/src/__support/freestore.h | 38 ++++++++++++++--------------------
1 file changed, 15 insertions(+), 23 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 543caf3f9ca31..4b5905d6e92f4 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -24,6 +24,7 @@
#include "src/__support/freetrie.h"
#include "src/__support/macros/config.h"
#include "src/__support/macros/optimization.h"
+#include "src/__support/math_extras.h"
namespace LIBC_NAMESPACE_DECL {
@@ -110,14 +111,11 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
+ // Minimal available size for allocation.
static constexpr size_t MIN_INNER_SIZE =
MIN_OUTER_SIZE - BlockRef::HEADER_SIZE + BlockRef::PREV_FIELD_SIZE;
- static constexpr size_t LINEAR_BINS =
- (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN)
- ? ((EXP_BASE << UNIT_SIZE_LOG2) - MIN_INNER_SIZE) /
- CONFIG::UNIT_SIZE +
- 1
- : EXP_BASE;
+ // Number of bins grows linearly.
+ static constexpr size_t LINEAR_BINS = EXP_BASE + 1;
LIBC_INLINE TLSFFreeStoreImpl() = default;
LIBC_INLINE TLSFFreeStoreImpl(const TLSFFreeStoreImpl &other) = delete;
@@ -155,14 +153,12 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
template <typename CONFIG>
LIBC_INLINE constexpr size_t
TLSFFreeStoreImpl<CONFIG>::size_to_bit_index(size_t size) {
- if (size <= (EXP_BASE << UNIT_SIZE_LOG2)) {
- if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
- if (size <= MIN_INNER_SIZE)
- return 0;
- return ((size - MIN_INNER_SIZE - 1) >> UNIT_SIZE_LOG2) + 1;
- }
- return size >> UNIT_SIZE_LOG2;
- }
+ // Compute bin as delta on top of min_inner_size
+ if (sub_overflow(size, MIN_INNER_SIZE + 1, size))
+ return 0;
+
+ if (size < (EXP_BASE << UNIT_SIZE_LOG2))
+ return (size >> UNIT_SIZE_LOG2) + 1;
size_t size_ilog2 = static_cast<size_t>(cpp::bit_width(size) - 1);
size_t exp_offset = (size_ilog2 - UNIT_SIZE_LOG2 - EXP_BASE_LOG2 - 1)
@@ -220,14 +216,10 @@ TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
template <typename CONFIG>
LIBC_INLINE constexpr size_t
TLSFFreeStoreImpl<CONFIG>::index_to_min_size(size_t index) {
- if (index < LINEAR_BINS) {
- if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
- if (index == 0)
- return 0;
- return MIN_INNER_SIZE + (index - 1) * CONFIG::UNIT_SIZE + 1;
- }
- return index << UNIT_SIZE_LOG2;
- }
+ if (index == 0)
+ return 0;
+ if (index < LINEAR_BINS)
+ return MIN_INNER_SIZE + 1 + ((index - 1) << UNIT_SIZE_LOG2);
size_t local_index = index - LINEAR_BINS;
size_t exp_index = local_index >> CONFIG::NUM_STEP_BITS;
@@ -235,7 +227,7 @@ TLSFFreeStoreImpl<CONFIG>::index_to_min_size(size_t index) {
size_t row_base = (EXP_BASE << exp_index) << UNIT_SIZE_LOG2;
size_t step_size = (STEP_SIZE << exp_index) << UNIT_SIZE_LOG2;
- return row_base + linear_index * step_size;
+ return MIN_INNER_SIZE + 1 + row_base + linear_index * step_size;
}
template <typename CONFIG>
>From 70ec06b64a0a738eb7d5759628ca63611e4cb709 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Mon, 3 Aug 2026 13:11:15 -0700
Subject: [PATCH 11/14] [libc] clean up UBs by using explicit storage
TAG=agy
CONV=e64ff65b-c845-4136-9173-da6f615197ee
---
libc/src/__support/freestore.h | 16 +++++++++++++---
libc/test/src/__support/freestore_test.cpp | 17 +++++++++++++++++
2 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 4b5905d6e92f4..51022c4961656 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -19,6 +19,7 @@
#include "src/__support/CPP/array.h"
#include "src/__support/CPP/bit.h"
#include "src/__support/CPP/limits.h"
+#include "src/__support/CPP/type_traits/bool_constant.h"
#include "src/__support/block.h"
#include "src/__support/freelist.h"
#include "src/__support/freetrie.h"
@@ -108,6 +109,12 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_OVERFLOW_BIN;
+private:
+ LIBC_INLINE constexpr TLSFFreeStoreImpl(cpp::bool_constant<true>)
+ : trie() {}
+ LIBC_INLINE constexpr TLSFFreeStoreImpl(cpp::bool_constant<false>)
+ : overflow_list() {}
+
public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
@@ -117,7 +124,8 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
// Number of bins grows linearly.
static constexpr size_t LINEAR_BINS = EXP_BASE + 1;
- LIBC_INLINE TLSFFreeStoreImpl() = default;
+ LIBC_INLINE constexpr TLSFFreeStoreImpl()
+ : TLSFFreeStoreImpl(cpp::bool_constant<USE_TRIE>{}) {}
LIBC_INLINE TLSFFreeStoreImpl(const TLSFFreeStoreImpl &other) = delete;
LIBC_INLINE TLSFFreeStoreImpl &
operator=(const TLSFFreeStoreImpl &other) = delete;
@@ -139,8 +147,10 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
cpp::array<FreeList, TOTAL_BITS - 1> free_lists{};
- FreeTrie trie{};
- FreeList overflow_list{};
+ union {
+ FreeTrie trie;
+ FreeList overflow_list;
+ };
LIBC_INLINE void set_bit(size_t bit_index);
LIBC_INLINE void clear_bit(size_t bit_index);
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index e4ee44d2776eb..41213468d7ff5 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -137,3 +137,20 @@ TEST(LlvmLibcFreeStore, IndexToMinSize) {
prev_size = min_size;
}
}
+
+struct NoTrieConfig : public LIBC_NAMESPACE::DefaultFreeStoreConfig {
+ static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN = false;
+};
+
+TEST(LlvmLibcFreeStore, NoTrieOverflow) {
+ LIBC_NAMESPACE::TLSFFreeStore<NoTrieConfig> store;
+ store.set_range({0, 4096});
+ byte mem[1024];
+ optional<BlockRef> maybeBlock = BlockRef::init(mem);
+ ASSERT_TRUE(maybeBlock.has_value());
+ BlockRef block = *maybeBlock;
+ store.insert(block);
+ BlockRef result = store.remove_best_fit(block.inner_size());
+ EXPECT_EQ(result.addr(), block.addr());
+}
+
>From 34242a243e748c2d5631818c96b1eb352c5d8de2 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Mon, 3 Aug 2026 13:18:38 -0700
Subject: [PATCH 12/14] address CR
---
libc/src/__support/freestore.h | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 51022c4961656..ccfae4343a08b 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -82,7 +82,7 @@ struct DefaultFreeStoreConfig {
// T |(Base 4K)| [4096 - 5119] | [5120 - 6143] | [6144 - 7167] | [7168 - 8191] |
// I +---------+---------------+---------------+---------------+---------------+
// A | Row = 3 | 8192 B | 10240 B | 12288 B | 14336 B |
-// L |(Base 8K)|[8192 - 10239]|[10240 - 12287]|[12288 - 14335]|[14336 - 16383]|
+// L |(Base 8K)|[8192 - 10239] |[10240 - 12287]|[12288 - 14335]|[14336 - 16383]|
// +---------+---------------+---------------+---------------+---------------+
//
// Note: For the real implementation, we don't actually store the lists in a
@@ -110,8 +110,7 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_OVERFLOW_BIN;
private:
- LIBC_INLINE constexpr TLSFFreeStoreImpl(cpp::bool_constant<true>)
- : trie() {}
+ LIBC_INLINE constexpr TLSFFreeStoreImpl(cpp::bool_constant<true>) : trie() {}
LIBC_INLINE constexpr TLSFFreeStoreImpl(cpp::bool_constant<false>)
: overflow_list() {}
@@ -145,8 +144,8 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
return block.outer_size() < MIN_OUTER_SIZE;
}
- cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
- cpp::array<FreeList, TOTAL_BITS - 1> free_lists{};
+ cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table;
+ cpp::array<FreeList, TOTAL_BITS - 1> free_lists;
union {
FreeTrie trie;
FreeList overflow_list;
>From 34553ae4893e908c99912e258a6a06ae95b1dbbc Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Mon, 3 Aug 2026 13:41:53 -0700
Subject: [PATCH 13/14] [libc] separate out TLSFTable to its own header and add
unit tests
Address code review comments:
- Extract TLSFTable abstraction into tlsf_table.h to encapsulate occupancy bitmaps and bin sizing formulas without changing core allocator algorithm logic or O(1) fast paths.
- Add exhaustive unit tests in tlsf_table_test.cpp.
- Remove redundant {} default member initializers from structured list classes.
- Format heap-related files with clang-format.
TAG=agy
CONV=e64ff65b-c845-4136-9173-da6f615197ee
---
libc/src/__support/CMakeLists.txt | 16 ++
libc/src/__support/freestore.h | 215 +++-----------------
libc/src/__support/tlsf_table.h | 205 +++++++++++++++++++
libc/test/src/__support/CMakeLists.txt | 10 +
libc/test/src/__support/freestore_test.cpp | 1 -
libc/test/src/__support/tlsf_table_test.cpp | 75 +++++++
6 files changed, 336 insertions(+), 186 deletions(-)
create mode 100644 libc/src/__support/tlsf_table.h
create mode 100644 libc/test/src/__support/tlsf_table_test.cpp
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index f4cb283976bc3..edea7446c6675 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -51,12 +51,28 @@ add_object_library(
.freelist
)
+add_header_library(
+ tlsf_table
+ HDRS
+ tlsf_table.h
+ DEPENDS
+ .block
+ .freelist
+ libc.src.__support.CPP.array
+ libc.src.__support.CPP.bit
+ libc.src.__support.CPP.limits
+ libc.src.__support.macros.config
+ libc.src.__support.macros.optimization
+ libc.src.__support.math_extras
+)
+
add_header_library(
freestore
HDRS
freestore.h
DEPENDS
.freetrie
+ .tlsf_table
)
libc_set_definition(libc_freelist_malloc_size "LIBC_FREELIST_MALLOC_SIZE=${LIBC_CONF_FREELIST_MALLOC_BUFFER_SIZE}")
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index ccfae4343a08b..562d3fa28b044 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -17,96 +17,28 @@
#include "hdr/stdint_proxy.h"
#include "hdr/types/size_t.h"
#include "src/__support/CPP/array.h"
-#include "src/__support/CPP/bit.h"
-#include "src/__support/CPP/limits.h"
#include "src/__support/CPP/type_traits/bool_constant.h"
#include "src/__support/block.h"
#include "src/__support/freelist.h"
#include "src/__support/freetrie.h"
#include "src/__support/macros/config.h"
#include "src/__support/macros/optimization.h"
-#include "src/__support/math_extras.h"
+#include "src/__support/tlsf_table.h"
namespace LIBC_NAMESPACE_DECL {
-/// Default configuration for TLSFFreeStore.
-struct DefaultFreeStoreConfig {
- static constexpr size_t UNIT_SIZE = BlockRef::MIN_ALIGN;
- static constexpr size_t STEP_SIZE_BITS = 3;
- static constexpr size_t NUM_STEP_BITS = 2;
- static constexpr size_t NUM_TABLE_ENTRIES = sizeof(uintptr_t) == 8 ? 3 : 6;
- static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN = true;
- static constexpr size_t LINEAR_SCAN_LIMIT = 16;
-};
-
// A two-level segregated fit store for free blocks.
//
-// The store starts with small lists that grow linearly for small sizes, which
-// covers [0, ... UNIT_SIZE * EXP_BASE]. For larger sizes, the bits are managed
-// in a 2-D table. One can think of each row containing NUM_STEPS lists. Along
-// the row, the size grows by 2 exponentially; along the column, the size
-// increases by STEP_SIZE linearly.
-//
-// Mathematical layout:
-// STEP_SIZE = 1 << STEP_SIZE_BITS
-// NUM_STEPS = 1 << NUM_STEP_BITS
-// EXP_BASE = STEP_SIZE * NUM_STEPS
-// LARGE_SIZE_THRESHOLD = UNIT_SIZE * EXP_BASE
-//
-// Visual representation with example parameters:
-// UNIT_SIZE = 32, STEP_SIZE = 8, NUM_STEPS = 4
-// EXP_BASE = 32, THRESHOLD = 1024 B (1 KiB)
-//
-// 1. Small Sizes (Linear Array):
-// Covers [0, ... 1024 B] growing directly by UNIT_SIZE = 32 B
-// +-------+-------+-------+-------+-------+-----------+---------------+
-// | [0 B] | [32B] | [64B] | [96B] | ... | [992 B] | [1024 B (Th)] |
-// +-------+-------+-------+-------+-------+-----------+---------------+
-//
-// 2. Large Sizes (2-D Table):
-// Rows = FL (Exponential growth), Columns = SL (Linear steps)
-// One can think of each Row containing NUM_STEPS (4) lists.
-//
-// LINEAR INCREASE ALONG COLUMN (SL) --->
-// +---------------+---------------+---------------+---------------+
-// | Col = 0 | Col = 1 | Col = 2 | Col = 3 |
-// | (Base) | (+25% FL) | (+50% FL) | (+75% FL) |
-// +---------+---------------+---------------+---------------+---------------+
-// E | Row = 0 | 1024 B | 1280 B | 1536 B | 1792 B |
-// X |(Base 1K)| [1024 - 1279] | [1280 - 1535] | [1536 - 1791] | [1792 - 2047] |
-// P +---------+---------------+---------------+---------------+---------------+
-// O | Row = 1 | 2048 B | 2560 B | 3072 B | 3584 B |
-// N |(Base 2K)| [2048 - 2559] | [2560 - 3071] | [3072 - 3583] | [3584 - 4095] |
-// E +---------+---------------+---------------+---------------+---------------+
-// N | Row = 2 | 4096 B | 5120 B | 6144 B | 7168 B |
-// T |(Base 4K)| [4096 - 5119] | [5120 - 6143] | [6144 - 7167] | [7168 - 8191] |
-// I +---------+---------------+---------------+---------------+---------------+
-// A | Row = 3 | 8192 B | 10240 B | 12288 B | 14336 B |
-// L |(Base 8K)|[8192 - 10239] |[10240 - 12287]|[12288 - 14335]|[14336 - 16383]|
-// +---------+---------------+---------------+---------------+---------------+
-//
-// Note: For the real implementation, we don't actually store the lists in a
-// 2-D structure. Instead, we flatten the entire 2-D layout into a single
-// flat 1-D array of size TOTAL_BITS (free_lists), and map sizes directly to
-// a continuous 1-D index using size_to_bit_index. The allocation state is
-// tracked compactly in the lookup_table bitmask array.
+// See TLSFTable in tlsf_table.h for the mathematical layout and bin mapping
+// logic.
template <typename CONFIG> class TLSFFreeStoreImpl {
-protected:
- static_assert(cpp::has_single_bit(CONFIG::UNIT_SIZE),
- "unit size must be a power of two");
- static_assert(CONFIG::NUM_TABLE_ENTRIES > 0,
- "the lookup table must have at least one entry");
-
- static constexpr size_t STEP_SIZE = size_t(1) << CONFIG::STEP_SIZE_BITS;
- static constexpr size_t NUM_STEPS = size_t(1) << CONFIG::NUM_STEP_BITS;
- static constexpr size_t EXP_BASE = STEP_SIZE * NUM_STEPS;
- static constexpr int UNIT_SIZE_LOG2 = cpp::bit_width(CONFIG::UNIT_SIZE) - 1;
- static constexpr int EXP_BASE_LOG2 =
- CONFIG::STEP_SIZE_BITS + CONFIG::NUM_STEP_BITS;
- static constexpr size_t BITS_PER_ENTRY =
- cpp::numeric_limits<uintptr_t>::digits;
- static constexpr size_t TOTAL_BITS =
- CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
+public:
+ using Table = TLSFTable<CONFIG>;
+ static constexpr size_t TOTAL_BITS = Table::TOTAL_BINS;
+ static constexpr size_t TOTAL_BINS = Table::TOTAL_BINS;
+ static constexpr size_t MIN_OUTER_SIZE = Table::MIN_OUTER_SIZE;
+ static constexpr size_t MIN_INNER_SIZE = Table::MIN_INNER_SIZE;
+ static constexpr size_t LINEAR_BINS = Table::LINEAR_BINS;
static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_OVERFLOW_BIN;
private:
@@ -115,21 +47,15 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
: overflow_list() {}
public:
- static constexpr size_t MIN_OUTER_SIZE = align_up(
- BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
- // Minimal available size for allocation.
- static constexpr size_t MIN_INNER_SIZE =
- MIN_OUTER_SIZE - BlockRef::HEADER_SIZE + BlockRef::PREV_FIELD_SIZE;
- // Number of bins grows linearly.
- static constexpr size_t LINEAR_BINS = EXP_BASE + 1;
-
LIBC_INLINE constexpr TLSFFreeStoreImpl()
: TLSFFreeStoreImpl(cpp::bool_constant<USE_TRIE>{}) {}
LIBC_INLINE TLSFFreeStoreImpl(const TLSFFreeStoreImpl &other) = delete;
LIBC_INLINE TLSFFreeStoreImpl &
operator=(const TLSFFreeStoreImpl &other) = delete;
- LIBC_INLINE static constexpr size_t index_to_min_size(size_t index);
+ LIBC_INLINE static constexpr size_t index_to_min_size(size_t index) {
+ return Table::bin_to_min_size(index);
+ }
LIBC_INLINE void set_range(FreeTrie::SizeRange range);
LIBC_INLINE void insert(BlockRef block);
LIBC_INLINE void remove(BlockRef block);
@@ -137,108 +63,26 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
return find_and_remove_fit(size);
}
LIBC_INLINE BlockRef find_and_remove_fit(size_t size);
- LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
+ LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size) {
+ return Table::size_to_bin(size);
+ }
protected:
LIBC_INLINE static bool too_small(BlockRef block) {
return block.outer_size() < MIN_OUTER_SIZE;
}
- cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table;
+ Table free_sizes;
cpp::array<FreeList, TOTAL_BITS - 1> free_lists;
union {
FreeTrie trie;
FreeList overflow_list;
};
- LIBC_INLINE void set_bit(size_t bit_index);
- LIBC_INLINE void clear_bit(size_t bit_index);
- LIBC_INLINE bool get_bit(size_t bit_index) const;
- LIBC_INLINE size_t find_first_bit_set_after(size_t bit_index) const;
LIBC_INLINE BlockRef remove_first_fit_in_list(size_t index, size_t size);
LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t size);
};
-template <typename CONFIG>
-LIBC_INLINE constexpr size_t
-TLSFFreeStoreImpl<CONFIG>::size_to_bit_index(size_t size) {
- // Compute bin as delta on top of min_inner_size
- if (sub_overflow(size, MIN_INNER_SIZE + 1, size))
- return 0;
-
- if (size < (EXP_BASE << UNIT_SIZE_LOG2))
- return (size >> UNIT_SIZE_LOG2) + 1;
-
- size_t size_ilog2 = static_cast<size_t>(cpp::bit_width(size) - 1);
- size_t exp_offset = (size_ilog2 - UNIT_SIZE_LOG2 - EXP_BASE_LOG2 - 1)
- << CONFIG::NUM_STEP_BITS;
- size_t step_index = size >> (size_ilog2 - CONFIG::NUM_STEP_BITS);
- size_t index = LINEAR_BINS + exp_offset + step_index;
-
- return index < TOTAL_BITS ? index : TOTAL_BITS - 1;
-}
-
-template <typename CONFIG>
-LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::set_bit(size_t bit_index) {
- size_t entry_index = bit_index / BITS_PER_ENTRY;
- size_t bit_offset = bit_index % BITS_PER_ENTRY;
- lookup_table[entry_index] |= uintptr_t(1) << bit_offset;
-}
-
-template <typename CONFIG>
-LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::clear_bit(size_t bit_index) {
- size_t entry_index = bit_index / BITS_PER_ENTRY;
- size_t bit_offset = bit_index % BITS_PER_ENTRY;
- lookup_table[entry_index] &= ~(uintptr_t(1) << bit_offset);
-}
-
-template <typename CONFIG>
-LIBC_INLINE bool TLSFFreeStoreImpl<CONFIG>::get_bit(size_t bit_index) const {
- size_t entry_index = bit_index / BITS_PER_ENTRY;
- size_t bit_offset = bit_index % BITS_PER_ENTRY;
- return (lookup_table[entry_index] & (uintptr_t(1) << bit_offset)) != 0;
-}
-
-template <typename CONFIG>
-LIBC_INLINE size_t
-TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
- if (bit_index >= TOTAL_BITS - 1)
- return TOTAL_BITS;
-
- size_t target_index = bit_index + 1;
- size_t start_entry = target_index / BITS_PER_ENTRY;
- size_t bit_offset = target_index % BITS_PER_ENTRY;
-
- uintptr_t value = lookup_table[start_entry] & (~uintptr_t(0) << bit_offset);
- if (value != 0)
- return start_entry * BITS_PER_ENTRY +
- static_cast<size_t>(cpp::countr_zero(value));
-
- for (size_t i = start_entry + 1; i < CONFIG::NUM_TABLE_ENTRIES; ++i) {
- value = lookup_table[i];
- if (value != 0)
- return i * BITS_PER_ENTRY + static_cast<size_t>(cpp::countr_zero(value));
- }
- return TOTAL_BITS;
-}
-
-template <typename CONFIG>
-LIBC_INLINE constexpr size_t
-TLSFFreeStoreImpl<CONFIG>::index_to_min_size(size_t index) {
- if (index == 0)
- return 0;
- if (index < LINEAR_BINS)
- return MIN_INNER_SIZE + 1 + ((index - 1) << UNIT_SIZE_LOG2);
-
- size_t local_index = index - LINEAR_BINS;
- size_t exp_index = local_index >> CONFIG::NUM_STEP_BITS;
- size_t linear_index = local_index & (NUM_STEPS - 1);
-
- size_t row_base = (EXP_BASE << exp_index) << UNIT_SIZE_LOG2;
- size_t step_size = (STEP_SIZE << exp_index) << UNIT_SIZE_LOG2;
- return MIN_INNER_SIZE + 1 + row_base + linear_index * step_size;
-}
-
template <typename CONFIG>
LIBC_INLINE void
TLSFFreeStoreImpl<CONFIG>::set_range(FreeTrie::SizeRange range) {
@@ -259,7 +103,7 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit_in_trie(size_t size) {
BlockRef block = best_fit->block();
trie.remove(best_fit);
if (trie.empty())
- clear_bit(TOTAL_BITS - 1);
+ free_sizes.mark_vacant(TOTAL_BITS - 1);
return block;
}
return BlockRef();
@@ -276,12 +120,12 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
trie.push(block);
else
overflow_list.push(block);
- set_bit(bit_index);
+ free_sizes.mark_occupied(bit_index);
return;
}
free_lists[bit_index].push(block);
- set_bit(bit_index);
+ free_sizes.mark_occupied(bit_index);
}
template <typename CONFIG>
@@ -294,12 +138,12 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
if constexpr (USE_TRIE) {
trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
if (trie.empty())
- clear_bit(bit_index);
+ free_sizes.mark_vacant(bit_index);
} else {
overflow_list.remove(
reinterpret_cast<FreeList::Node *>(block.usable_space()));
if (overflow_list.empty())
- clear_bit(bit_index);
+ free_sizes.mark_vacant(bit_index);
}
return;
}
@@ -307,7 +151,7 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
free_lists[bit_index].remove(
reinterpret_cast<FreeList::Node *>(block.usable_space()));
if (free_lists[bit_index].empty())
- clear_bit(bit_index);
+ free_sizes.mark_vacant(bit_index);
}
template <typename CONFIG>
@@ -325,7 +169,7 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
if (cur->size() >= size) {
list.remove(cur);
if (list.empty())
- clear_bit(index);
+ free_sizes.mark_vacant(index);
return cur->block();
}
cur = cur->next();
@@ -343,11 +187,12 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
// Fast path for small linear bins if UNIT_SIZE == MIN_ALIGN
if constexpr (CONFIG::UNIT_SIZE == BlockRef::MIN_ALIGN) {
- if (LIBC_LIKELY(bit_index < LINEAR_BINS && get_bit(bit_index))) {
+ if (LIBC_LIKELY(bit_index < LINEAR_BINS &&
+ free_sizes.is_occupied(bit_index))) {
BlockRef block = free_lists[bit_index].front();
free_lists[bit_index].pop();
if (free_lists[bit_index].empty())
- clear_bit(bit_index);
+ free_sizes.mark_vacant(bit_index);
return block;
}
}
@@ -360,7 +205,7 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
}
// 1. Try oversized bins (guaranteed fit, but larger).
- size_t oversized_bit = find_first_bit_set_after(bit_index);
+ size_t oversized_bit = free_sizes.find_first_occupied_after(bit_index);
if (LIBC_LIKELY(oversized_bit < TOTAL_BITS)) {
if (LIBC_UNLIKELY(oversized_bit == TOTAL_BITS - 1)) {
if constexpr (USE_TRIE)
@@ -372,12 +217,12 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
BlockRef block = free_lists[oversized_bit].front();
free_lists[oversized_bit].pop();
if (free_lists[oversized_bit].empty())
- clear_bit(oversized_bit);
+ free_sizes.mark_vacant(oversized_bit);
return block;
}
// 2. Try exact fit (fallback).
- if (get_bit(bit_index)) {
+ if (free_sizes.is_occupied(bit_index)) {
if (BlockRef block = remove_first_fit_in_list(bit_index, size))
return block;
}
diff --git a/libc/src/__support/tlsf_table.h b/libc/src/__support/tlsf_table.h
new file mode 100644
index 0000000000000..1b4be90c2ee50
--- /dev/null
+++ b/libc/src/__support/tlsf_table.h
@@ -0,0 +1,205 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains a two-level segregated fit table and mapping helper.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_TLSF_TABLE_H
+#define LLVM_LIBC_SRC___SUPPORT_TLSF_TABLE_H
+
+#include "hdr/stdint_proxy.h"
+#include "hdr/types/size_t.h"
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/block.h"
+#include "src/__support/freelist.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
+#include "src/__support/math_extras.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+/// Default configuration for TLSFFreeStore and TLSFTable.
+struct DefaultFreeStoreConfig {
+ static constexpr size_t UNIT_SIZE = BlockRef::MIN_ALIGN;
+ static constexpr size_t STEP_SIZE_BITS = 3;
+ static constexpr size_t NUM_STEP_BITS = 2;
+ static constexpr size_t NUM_TABLE_ENTRIES = sizeof(uintptr_t) == 8 ? 3 : 6;
+ static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN = true;
+ static constexpr size_t LINEAR_SCAN_LIMIT = 16;
+};
+
+// A two-level segregated fit occupancy table and mapping helper.
+//
+// The table starts with small bins that grow linearly for small sizes, which
+// covers [0, ... UNIT_SIZE * EXP_BASE]. For larger sizes, the bits are managed
+// in a 2-D table. One can think of each row containing NUM_STEPS lists. Along
+// the row, the size grows by 2 exponentially; along the column, the size
+// increases by STEP_SIZE linearly.
+//
+// Mathematical layout:
+// STEP_SIZE = 1 << STEP_SIZE_BITS
+// NUM_STEPS = 1 << NUM_STEP_BITS
+// EXP_BASE = STEP_SIZE * NUM_STEPS
+// LARGE_SIZE_THRESHOLD = UNIT_SIZE * EXP_BASE
+//
+// Visual representation with example parameters:
+// UNIT_SIZE = 32, STEP_SIZE = 8, NUM_STEPS = 4
+// EXP_BASE = 32, THRESHOLD = 1024 B (1 KiB)
+//
+// 1. Small Sizes (Linear Array):
+// Covers [0, ... 1024 B] growing directly by UNIT_SIZE = 32 B
+// +-------+-------+-------+-------+-------+-----------+---------------+
+// | [0 B] | [32B] | [64B] | [96B] | ... | [992 B] | [1024 B (Th)] |
+// +-------+-------+-------+-------+-------+-----------+---------------+
+//
+// 2. Large Sizes (2-D Table):
+// Rows = FL (Exponential growth), Columns = SL (Linear steps)
+// One can think of each Row containing NUM_STEPS (4) lists.
+//
+// LINEAR INCREASE ALONG COLUMN (SL) --->
+// +---------------+---------------+---------------+---------------+
+// | Col = 0 | Col = 1 | Col = 2 | Col = 3 |
+// | (Base) | (+25% FL) | (+50% FL) | (+75% FL) |
+// +---------+---------------+---------------+---------------+---------------+
+// E | Row = 0 | 1024 B | 1280 B | 1536 B | 1792 B |
+// X |(Base 1K)| [1024 - 1279] | [1280 - 1535] | [1536 - 1791] | [1792 - 2047] |
+// P +---------+---------------+---------------+---------------+---------------+
+// O | Row = 1 | 2048 B | 2560 B | 3072 B | 3584 B |
+// N |(Base 2K)| [2048 - 2559] | [2560 - 3071] | [3072 - 3583] | [3584 - 4095] |
+// E +---------+---------------+---------------+---------------+---------------+
+// N | Row = 2 | 4096 B | 5120 B | 6144 B | 7168 B |
+// T |(Base 4K)| [4096 - 5119] | [5120 - 6143] | [6144 - 7167] | [7168 - 8191] |
+// I +---------+---------------+---------------+---------------+---------------+
+// A | Row = 3 | 8192 B | 10240 B | 12288 B | 14336 B |
+// L |(Base 8K)|[8192 - 10239] |[10240 - 12287]|[12288 - 14335]|[14336 - 16383]|
+// +---------+---------------+---------------+---------------+---------------+
+//
+// Note: For the real implementation, we don't actually store the lists in a
+// 2-D structure. Instead, we flatten the entire 2-D layout into a single
+// flat 1-D array of size TOTAL_BINS, and map sizes directly to a continuous
+// 1-D index using size_to_bin. The occupancy state is tracked compactly in
+// the lookup_table bitmask array.
+template <typename CONFIG> class TLSFTable {
+protected:
+ static_assert(cpp::has_single_bit(CONFIG::UNIT_SIZE),
+ "unit size must be a power of two");
+ static_assert(CONFIG::NUM_TABLE_ENTRIES > 0,
+ "the lookup table must have at least one entry");
+
+ static constexpr size_t STEP_SIZE = size_t(1) << CONFIG::STEP_SIZE_BITS;
+ static constexpr size_t NUM_STEPS = size_t(1) << CONFIG::NUM_STEP_BITS;
+ static constexpr size_t EXP_BASE = STEP_SIZE * NUM_STEPS;
+ static constexpr int UNIT_SIZE_LOG2 = cpp::bit_width(CONFIG::UNIT_SIZE) - 1;
+ static constexpr int EXP_BASE_LOG2 =
+ CONFIG::STEP_SIZE_BITS + CONFIG::NUM_STEP_BITS;
+ static constexpr size_t BITS_PER_ENTRY =
+ cpp::numeric_limits<uintptr_t>::digits;
+
+public:
+ static constexpr size_t TOTAL_BINS =
+ CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
+ static constexpr size_t TOTAL_BITS = TOTAL_BINS;
+
+ static constexpr size_t MIN_OUTER_SIZE = align_up(
+ BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
+ // Minimal available size for allocation.
+ static constexpr size_t MIN_INNER_SIZE =
+ MIN_OUTER_SIZE - BlockRef::HEADER_SIZE + BlockRef::PREV_FIELD_SIZE;
+ // Number of bins grows linearly.
+ static constexpr size_t LINEAR_BINS = EXP_BASE + 1;
+
+ LIBC_INLINE constexpr TLSFTable() = default;
+
+ LIBC_INLINE static constexpr size_t size_to_bin(size_t size);
+ LIBC_INLINE static constexpr size_t bin_to_min_size(size_t bin);
+
+ LIBC_INLINE void mark_occupied(size_t bin) {
+ size_t entry_index = bin / BITS_PER_ENTRY;
+ size_t bit_offset = bin % BITS_PER_ENTRY;
+ lookup_table[entry_index] |= uintptr_t(1) << bit_offset;
+ }
+
+ LIBC_INLINE void mark_vacant(size_t bin) {
+ size_t entry_index = bin / BITS_PER_ENTRY;
+ size_t bit_offset = bin % BITS_PER_ENTRY;
+ lookup_table[entry_index] &= ~(uintptr_t(1) << bit_offset);
+ }
+
+ LIBC_INLINE bool is_occupied(size_t bin) const {
+ size_t entry_index = bin / BITS_PER_ENTRY;
+ size_t bit_offset = bin % BITS_PER_ENTRY;
+ return (lookup_table[entry_index] & (uintptr_t(1) << bit_offset)) != 0;
+ }
+
+ LIBC_INLINE size_t find_first_occupied_after(size_t bin) const {
+ if (bin >= TOTAL_BINS - 1)
+ return TOTAL_BINS;
+
+ size_t target_index = bin + 1;
+ size_t start_entry = target_index / BITS_PER_ENTRY;
+ size_t bit_offset = target_index % BITS_PER_ENTRY;
+
+ uintptr_t value = lookup_table[start_entry] & (~uintptr_t(0) << bit_offset);
+ if (value != 0)
+ return start_entry * BITS_PER_ENTRY +
+ static_cast<size_t>(cpp::countr_zero(value));
+
+ for (size_t i = start_entry + 1; i < CONFIG::NUM_TABLE_ENTRIES; ++i) {
+ value = lookup_table[i];
+ if (value != 0)
+ return i * BITS_PER_ENTRY +
+ static_cast<size_t>(cpp::countr_zero(value));
+ }
+ return TOTAL_BINS;
+ }
+
+private:
+ cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
+};
+
+template <typename CONFIG>
+LIBC_INLINE constexpr size_t TLSFTable<CONFIG>::size_to_bin(size_t size) {
+ // Compute bin as delta on top of min_inner_size
+ if (sub_overflow(size, MIN_INNER_SIZE + 1, size))
+ return 0;
+
+ if (size < (EXP_BASE << UNIT_SIZE_LOG2))
+ return (size >> UNIT_SIZE_LOG2) + 1;
+
+ size_t size_ilog2 = static_cast<size_t>(cpp::bit_width(size) - 1);
+ size_t exp_offset = (size_ilog2 - UNIT_SIZE_LOG2 - EXP_BASE_LOG2 - 1)
+ << CONFIG::NUM_STEP_BITS;
+ size_t step_index = size >> (size_ilog2 - CONFIG::NUM_STEP_BITS);
+ size_t index = LINEAR_BINS + exp_offset + step_index;
+
+ return index < TOTAL_BINS ? index : TOTAL_BINS - 1;
+}
+
+template <typename CONFIG>
+LIBC_INLINE constexpr size_t TLSFTable<CONFIG>::bin_to_min_size(size_t bin) {
+ if (bin == 0)
+ return 0;
+ if (bin < LINEAR_BINS)
+ return MIN_INNER_SIZE + 1 + ((bin - 1) << UNIT_SIZE_LOG2);
+
+ size_t local_index = bin - LINEAR_BINS;
+ size_t exp_index = local_index >> CONFIG::NUM_STEP_BITS;
+ size_t linear_index = local_index & (NUM_STEPS - 1);
+
+ size_t row_base = (EXP_BASE << exp_index) << UNIT_SIZE_LOG2;
+ size_t step_size = (STEP_SIZE << exp_index) << UNIT_SIZE_LOG2;
+ return MIN_INNER_SIZE + 1 + row_base + linear_index * step_size;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_TLSF_TABLE_H
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 8233529266326..93650e7fdb18e 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -53,6 +53,16 @@ if(NOT LIBC_TARGET_OS_IS_GPU)
libc.src.__support.freestore
libc.src.__support.freetrie
)
+
+ add_libc_test(
+ tlsf_table_test
+ SUITE
+ libc-support-tests
+ SRCS
+ tlsf_table_test.cpp
+ DEPENDS
+ libc.src.__support.tlsf_table
+ )
endif()
# TODO: FreeListHeap uses the _end symbol which conflicts with the _end symbol
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index 41213468d7ff5..9e6be63c410d5 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -153,4 +153,3 @@ TEST(LlvmLibcFreeStore, NoTrieOverflow) {
BlockRef result = store.remove_best_fit(block.inner_size());
EXPECT_EQ(result.addr(), block.addr());
}
-
diff --git a/libc/test/src/__support/tlsf_table_test.cpp b/libc/test/src/__support/tlsf_table_test.cpp
new file mode 100644
index 0000000000000..01c6de4c5d762
--- /dev/null
+++ b/libc/test/src/__support/tlsf_table_test.cpp
@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for a TLSFTable.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/tlsf_table.h"
+#include "test/UnitTest/Test.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+using Table = TLSFTable<DefaultFreeStoreConfig>;
+
+TEST(LlvmLibcTLSFTableTest, SizeToBin) {
+ EXPECT_EQ(Table::size_to_bin(0), static_cast<size_t>(0));
+ EXPECT_EQ(Table::size_to_bin(Table::MIN_INNER_SIZE), static_cast<size_t>(0));
+ EXPECT_EQ(Table::size_to_bin(Table::MIN_INNER_SIZE + 1),
+ static_cast<size_t>(1));
+}
+
+TEST(LlvmLibcTLSFTableTest, BinToMinSize) {
+ EXPECT_EQ(Table::bin_to_min_size(0), static_cast<size_t>(0));
+ EXPECT_EQ(Table::bin_to_min_size(1),
+ static_cast<size_t>(Table::MIN_INNER_SIZE + 1));
+
+ size_t prev_size = 0;
+ for (size_t i = 1; i < 64; ++i) {
+ size_t min_size = Table::bin_to_min_size(i);
+ EXPECT_GT(min_size, prev_size);
+ prev_size = min_size;
+ }
+}
+
+TEST(LlvmLibcTLSFTableTest, OccupancyQueriesAndMutations) {
+ Table table;
+ for (size_t i = 0; i < Table::TOTAL_BINS; ++i) {
+ EXPECT_FALSE(table.is_occupied(i));
+ }
+
+ table.mark_occupied(5);
+ EXPECT_TRUE(table.is_occupied(5));
+ EXPECT_FALSE(table.is_occupied(4));
+ EXPECT_FALSE(table.is_occupied(6));
+
+ table.mark_occupied(100);
+ EXPECT_TRUE(table.is_occupied(100));
+
+ table.mark_vacant(5);
+ EXPECT_FALSE(table.is_occupied(5));
+ EXPECT_TRUE(table.is_occupied(100));
+}
+
+TEST(LlvmLibcTLSFTableTest, FindFirstOccupiedAfter) {
+ Table table;
+ EXPECT_EQ(table.find_first_occupied_after(0), Table::TOTAL_BINS);
+ EXPECT_EQ(table.find_first_occupied_after(10), Table::TOTAL_BINS);
+
+ table.mark_occupied(10);
+ table.mark_occupied(65);
+
+ EXPECT_EQ(table.find_first_occupied_after(0), static_cast<size_t>(10));
+ EXPECT_EQ(table.find_first_occupied_after(9), static_cast<size_t>(10));
+ EXPECT_EQ(table.find_first_occupied_after(10), static_cast<size_t>(65));
+ EXPECT_EQ(table.find_first_occupied_after(64), static_cast<size_t>(65));
+ EXPECT_EQ(table.find_first_occupied_after(65), Table::TOTAL_BINS);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
>From d6e58d25bebea9650131a37e5725969d6be241e8 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Mon, 3 Aug 2026 14:47:11 -0700
Subject: [PATCH 14/14] improve docs
---
libc/src/__support/freestore.h | 22 +++++++++++
libc/src/__support/tlsf_table.h | 70 ++++++++++++++++++++-------------
2 files changed, 64 insertions(+), 28 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 562d3fa28b044..439afa6ea6621 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -31,6 +31,28 @@ namespace LIBC_NAMESPACE_DECL {
//
// See TLSFTable in tlsf_table.h for the mathematical layout and bin mapping
// logic.
+//
+// Generic memory workloads typically cluster around smaller allocation
+// requests, exhibiting an inverse relationship between request frequency
+// and block size. Furthermore, large allocations can suffer from severe
+// internal fragmentation if managed with overly coarse-grained binning.
+//
+// To accommodate these workload dynamics, this store organizes free blocks
+// into three specialized tiers, each employing a tailored management strategy:
+//
+// 1. Small Blocks (Exact-Fit Fast Path): Small blocks are maintained in linear,
+// exact-size bins. When a matching block is available, fast-path allocations
+// return an exact-fit block immediately in O(1) time without list traversal
+// or block splitting.
+// 2. Medium Blocks (Two-Level Segregated Fit Table): Mid-sized blocks are
+// managed across an exponential and linear bin grid. To preserve constant
+// O(1) allocation time on the fast path, the allocator preferentially pops
+// from an oversized bin first, accepting minor block splitting overhead to
+// prevent memory waste while guaranteeing bounded search latency.
+// 3. Large Blocks (Best-Fit Trie): For the coldest, largest block sizes, blocks
+// are stored in an ordered trie (when enabled). These allocations are
+// serviced using logarithmic best-fit searches, prioritizing space
+// efficiency and minimal fragmentation over immediate O(1) latency.
template <typename CONFIG> class TLSFFreeStoreImpl {
public:
using Table = TLSFTable<CONFIG>;
diff --git a/libc/src/__support/tlsf_table.h b/libc/src/__support/tlsf_table.h
index 1b4be90c2ee50..a302fba15f11c 100644
--- a/libc/src/__support/tlsf_table.h
+++ b/libc/src/__support/tlsf_table.h
@@ -39,48 +39,62 @@ struct DefaultFreeStoreConfig {
// A two-level segregated fit occupancy table and mapping helper.
//
-// The table starts with small bins that grow linearly for small sizes, which
-// covers [0, ... UNIT_SIZE * EXP_BASE]. For larger sizes, the bits are managed
-// in a 2-D table. One can think of each row containing NUM_STEPS lists. Along
-// the row, the size grows by 2 exponentially; along the column, the size
-// increases by STEP_SIZE linearly.
+// Rather than mapping raw block sizes directly, the table maps the size delta
+// over MIN_INNER_SIZE (delta = size - (MIN_INNER_SIZE + 1)). Small allocations
+// map directly to exact-size linear bins covering Bins 0 through EXP_BASE (33
+// linear bins total). Larger allocations are managed in a 2-D table starting at
+// Bin 33. One can think of each row containing NUM_STEPS lists. Along the row,
+// the delta grows exponentially by powers of 2; along the column, the delta
+// increases linearly by STEP_SIZE increments.
//
// Mathematical layout:
+// delta = size <= MIN_INNER_SIZE ? 0 : size - (MIN_INNER_SIZE + 1)
// STEP_SIZE = 1 << STEP_SIZE_BITS
// NUM_STEPS = 1 << NUM_STEP_BITS
// EXP_BASE = STEP_SIZE * NUM_STEPS
-// LARGE_SIZE_THRESHOLD = UNIT_SIZE * EXP_BASE
+// LINEAR_BINS = EXP_BASE + 1
+// LARGE_DELTA_THRESHOLD = UNIT_SIZE * EXP_BASE
//
-// Visual representation with example parameters:
-// UNIT_SIZE = 32, STEP_SIZE = 8, NUM_STEPS = 4
-// EXP_BASE = 32, THRESHOLD = 1024 B (1 KiB)
+// Visual representation with realistic configuration parameters:
+// MIN_INNER_SIZE = 24 B, UNIT_SIZE = 16, STEP_SIZE = 8, NUM_STEPS = 4
+// EXP_BASE = 32, LINEAR_BINS = 33, THRESHOLD = 512 B (Size 536 B)
//
-// 1. Small Sizes (Linear Array):
-// Covers [0, ... 1024 B] growing directly by UNIT_SIZE = 32 B
-// +-------+-------+-------+-------+-------+-----------+---------------+
-// | [0 B] | [32B] | [64B] | [96B] | ... | [992 B] | [1024 B (Th)] |
-// +-------+-------+-------+-------+-------+-----------+---------------+
+// 1. Small Sizes (Linear Bins on Delta over MIN_INNER_SIZE = 24 B):
+// Covers size range [0, ... 536 B] across Bins 0 to 32 (EXP_BASE = 32).
+// Assuming UNIT_SIZE matches block minimum alignment (step = unit = 16 B),
+// these linear bins hold blocks of exact inner sizes (e.g., Bin 0 = 24 B,
+// Bin 1 = 40 B, Bin 2 = 56 B). The byte ranges shown below reflect user
+// payload sizes that map directly into each exact-fit bin:
+// +-----------+-------------+-------------+-----+--------------+--------------+
+// | Bin 0 | Bin 1 | Bin 2 | ... | Bin 31 | Bin 32 |
+// | [0..24 B] | [25..40 B] | [41..56 B] | ... | [505..520 B] | [521..536 B] |
+// | (Delta 0) | (D: 0..15) | (D: 16..31) | ... | (D: 480..495)| (D: 496..511)|
+// +-----------+-------------+-------------+-----+--------------+--------------+
//
-// 2. Large Sizes (2-D Table):
+// 2. Large Sizes (2-D Table on Delta over MIN_INNER_SIZE for Sizes > 536 B):
// Rows = FL (Exponential growth), Columns = SL (Linear steps)
-// One can think of each Row containing NUM_STEPS (4) lists.
+// Each cell shows Bin index, exact Block Size range, and Delta (D) range:
//
// LINEAR INCREASE ALONG COLUMN (SL) --->
// +---------------+---------------+---------------+---------------+
// | Col = 0 | Col = 1 | Col = 2 | Col = 3 |
-// | (Base) | (+25% FL) | (+50% FL) | (+75% FL) |
+// | (Base) | (+25% Step) | (+50% Step) | (+75% Step) |
// +---------+---------------+---------------+---------------+---------------+
-// E | Row = 0 | 1024 B | 1280 B | 1536 B | 1792 B |
-// X |(Base 1K)| [1024 - 1279] | [1280 - 1535] | [1536 - 1791] | [1792 - 2047] |
-// P +---------+---------------+---------------+---------------+---------------+
-// O | Row = 1 | 2048 B | 2560 B | 3072 B | 3584 B |
-// N |(Base 2K)| [2048 - 2559] | [2560 - 3071] | [3072 - 3583] | [3584 - 4095] |
-// E +---------+---------------+---------------+---------------+---------------+
-// N | Row = 2 | 4096 B | 5120 B | 6144 B | 7168 B |
-// T |(Base 4K)| [4096 - 5119] | [5120 - 6143] | [6144 - 7167] | [7168 - 8191] |
-// I +---------+---------------+---------------+---------------+---------------+
-// A | Row = 3 | 8192 B | 10240 B | 12288 B | 14336 B |
-// L |(Base 8K)|[8192 - 10239] |[10240 - 12287]|[12288 - 14335]|[14336 - 16383]|
+// E | Row = 0 | Bin 33 | Bin 34 | Bin 35 | Bin 36 |
+// X | Base D: | [537..664 B] | [665..792 B] | [793..920 B] | [921..1048 B] |
+// P | 512 | D:[512..639] | D:[640..767] | D:[768..895] | D:[896..1023] |
+// +---------+---------------+---------------+---------------+---------------+
+// O | Row = 1 | Bin 37 | Bin 38 | Bin 39 | Bin 40 |
+// N | Base D: |[1049..1304 B] |[1305..1560 B] |[1561..1816 B] |[1817..2072 B] |
+// E | 1024 |D:[1024..1279] |D:[1280..1535] |D:[1536..1791] |D:[1792..2047] |
+// +---------+---------------+---------------+---------------+---------------+
+// N | Row = 2 | Bin 41 | Bin 42 | Bin 43 | Bin 44 |
+// T | Base D: |[2073..2584 B] |[2585..3096 B] |[3097..3608 B] |[3609..4120 B] |
+// I | 2048 |D:[2048..2559] |D:[2560..3071] |D:[3072..3583] |D:[3584..4095] |
+// +---------+---------------+---------------+---------------+---------------+
+// A | Row = 3 | Bin 45 | Bin 46 | Bin 47 | Bin 48 |
+// L | Base D: |[4121..5144 B] |[5145..6168 B] |[6169..7192 B] |[7193..8216 B] |
+// | 4096 |D:[4096..5119] |D:[5120..6143] |D:[6144..7167] |D:[7168..8191] |
// +---------+---------------+---------------+---------------+---------------+
//
// Note: For the real implementation, we don't actually store the lists in a
More information about the libc-commits
mailing list