[libc-commits] [libc] [llvm] [libc] make baremetal freelist heap headers-only (PR #205381)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Tue Jun 23 09:56:17 PDT 2026
https://github.com/SchrodingerZhu created https://github.com/llvm/llvm-project/pull/205381
This change moves all the implementation of FreeList, FreeTrie and FreeListHeap to headers, making it header-only. Outlined functions are marked as [[gnu::noinline]] to preserve their outlined behavior.
>From e7eed7b4e0600b9d62c6c44ebf355eb1bbfc2c3c Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 10 Jun 2026 13:50:35 -0700
Subject: [PATCH 1/3] Refactor FreeTrie and Implement TLSF FreeStore
- Make FreeTrie a proxy holding root by reference.
- Move root and range storage outside FreeTrie (into FreeStore).
- Add find_min and pop_min implementations.
- Update tests and add pop_min test.
- Replace freestore.h with TLSF implementation.
- Add USE_TRIE_FOR_LARGE_FREELIST option to use FreeTrie in TLSF.
- Remove obsolete set_range interface from FreeStore, FreeListHeap, and tests.
TAG=agy
CONV=fe3b4efa-7a5b-4c74-8257-e53f0d6e4850
---
libc/src/__support/freelist.h | 3 +
libc/src/__support/freelist_heap.h | 1 -
libc/src/__support/freestore.h | 403 +++++++++++++++++----
libc/src/__support/freetrie.h | 65 +++-
libc/test/src/__support/freestore_test.cpp | 27 +-
libc/test/src/__support/freetrie_test.cpp | 93 ++++-
6 files changed, 478 insertions(+), 114 deletions(-)
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index 48e70c7c29df6..ae2de684b3a24 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -41,6 +41,9 @@ class FreeList {
/// @returns The inner size of blocks in the list containing this node.
LIBC_INLINE size_t size() const { return block().inner_size(); }
+ /// @returns The next node in the list containing this node.
+ LIBC_INLINE Node *next_node() const { return next; }
+
private:
// Circularly linked pointers to adjacent nodes.
Node *prev;
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index a591d63ddd4e5..976a6eec4023e 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -84,7 +84,6 @@ LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
- free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
free_store.insert(block);
is_initialized = true;
}
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index adc0e061ace93..4fb90cabf47bd 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -7,114 +7,377 @@
//===----------------------------------------------------------------------===//
///
/// \file
-/// Interface for freestore.
+/// This file contains a two-level segregated fit free block store.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
#define LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
-#include "freetrie.h"
+#include "hdr/stdint_proxy.h"
+#include "hdr/types/size_t.h"
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/block.h"
+#include "src/__support/freelist.h"
+#include "src/__support/freetrie.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
namespace LIBC_NAMESPACE_DECL {
-/// A best-fit store of variously-sized free blocks. Blocks can be inserted and
-/// removed in logarithmic time.
-class FreeStore {
- friend class FreeListHeap;
-
-public:
- FreeStore() = default;
- FreeStore(const FreeStore &other) = delete;
- FreeStore &operator=(const FreeStore &other) = delete;
-
- /// Sets the range of possible block sizes. This can only be called when the
- /// trie is empty.
- LIBC_INLINE void set_range(FreeTrie::SizeRange range) {
- large_trie.set_range(range);
- }
-
- /// Insert a free block. If the block is too small to be tracked, nothing
- /// happens.
- void insert(BlockRef block);
+/// Configuration for TLSFFreeStore.
+template <size_t UNIT_SIZE_VAL, size_t STEP_SIZE_BITS_VAL,
+ size_t NUM_STEP_BITS_VAL, size_t NUM_TABLE_ENTRIES_VAL,
+ bool USE_TRIE_FOR_LARGE_FREELIST_VAL = false>
+struct TLSFFreeStoreConfig {
+ static constexpr size_t UNIT_SIZE = UNIT_SIZE_VAL;
+ static constexpr size_t STEP_SIZE_BITS = STEP_SIZE_BITS_VAL;
+ static constexpr size_t NUM_STEP_BITS = NUM_STEP_BITS_VAL;
+ static constexpr size_t NUM_TABLE_ENTRIES = NUM_TABLE_ENTRIES_VAL;
+ static constexpr bool USE_TRIE_FOR_LARGE_FREELIST =
+ USE_TRIE_FOR_LARGE_FREELIST_VAL;
+};
- /// Remove a free block. If the block is too small to be tracked, nothing
- /// happens.
- void remove(BlockRef block);
+// A two-level segregated fit store for free blocks.
+//
+// The store starts with small lists that grow linearly for small sizes, which
+// covers [0, ... UNIT_SIZE * EXP_BASE]. For larger sizes, the bits are managed
+// in a 2-D table. One can think of each row containing NUM_STEPS lists. Along
+// the row, the size grows by 2 exponentially; along the column, the size
+// increases by STEP_SIZE linearly.
+//
+// Mathematical layout:
+// STEP_SIZE = 1 << STEP_SIZE_BITS
+// NUM_STEPS = 1 << NUM_STEP_BITS
+// EXP_BASE = STEP_SIZE * NUM_STEPS
+// LARGE_SIZE_THRESHOLD = UNIT_SIZE * EXP_BASE
+//
+// Visual representation with example parameters:
+// UNIT_SIZE = 32, STEP_SIZE = 8, NUM_STEPS = 4
+// EXP_BASE = 32, THRESHOLD = 1024 B (1 KiB)
+//
+// 1. Small Sizes (Linear Array):
+// Covers [0, ... 1024 B] growing directly by UNIT_SIZE = 32 B
+// +-------+-------+-------+-------+-------+-----------+---------------+
+// | [0 B] | [32B] | [64B] | [96B] | ... | [992 B] | [1024 B (Th)] |
+// +-------+-------+-------+-------+-------+-----------+---------------+
+//
+// 2. Large Sizes (2-D Table):
+// Rows = FL (Exponential growth), Columns = SL (Linear steps)
+// One can think of each Row containing NUM_STEPS (4) lists.
+//
+// LINEAR INCREASE ALONG COLUMN (SL) --->
+// +---------------+---------------+---------------+---------------+
+// | Col = 0 | Col = 1 | Col = 2 | Col = 3 |
+// | (Base) | (+25% FL) | (+50% FL) | (+75% FL) |
+// +---------+---------------+---------------+---------------+---------------+
+// E | Row = 0 | 1024 B | 1280 B | 1536 B | 1792 B |
+// X |(Base 1K)| [1024 - 1279] | [1280 - 1535] | [1536 - 1791] | [1792 - 2047] |
+// P +---------+---------------+---------------+---------------+---------------+
+// O | Row = 1 | 2048 B | 2560 B | 3072 B | 3584 B |
+// N |(Base 2K)| [2048 - 2559] | [2560 - 3071] | [3072 - 3583] | [3584 - 4095] |
+// E +---------+---------------+---------------+---------------+---------------+
+// N | Row = 2 | 4096 B | 5120 B | 6144 B | 7168 B |
+// T |(Base 4K)| [4096 - 5119] | [5120 - 6143] | [6144 - 7167] | [7168 - 8191] |
+// I +---------+---------------+---------------+---------------+---------------+
+// A | Row = 3 | 8192 B | 10240 B | 12288 B | 14336 B |
+// L |(Base 8K)|[8192 - 10239]|[10240 - 12287]|[12288 - 14335]|[14336 - 16383]|
+// +---------+---------------+---------------+---------------+---------------+
+//
+// Note: For the real implementation, we don't actually store the lists in a
+// 2-D structure. Instead, we flatten the entire 2-D layout into a single
+// flat 1-D array of size TOTAL_BITS (free_lists), and map sizes directly to
+// a continuous 1-D index using size_to_bit_index. The allocation state is
+// tracked compactly in the lookup_table bitmask array.
+template <typename CONFIG> class TLSFFreeStoreImpl {
+protected:
+ static_assert(cpp::has_single_bit(CONFIG::UNIT_SIZE),
+ "unit size must be a power of two");
+ static_assert(CONFIG::NUM_TABLE_ENTRIES > 0,
+ "the lookup table must have at least one entry");
- /// Remove a best-fit free block that can contain the given size when
- /// allocated. Returns nullptr if there is no such block.
- BlockRef remove_best_fit(size_t size);
+ static constexpr size_t STEP_SIZE = size_t(1) << CONFIG::STEP_SIZE_BITS;
+ static constexpr size_t NUM_STEPS = size_t(1) << CONFIG::NUM_STEP_BITS;
+ static constexpr size_t EXP_BASE = STEP_SIZE * NUM_STEPS;
+ static constexpr int UNIT_SIZE_LOG2 = cpp::bit_width(CONFIG::UNIT_SIZE) - 1;
+ static constexpr int EXP_BASE_LOG2 =
+ CONFIG::STEP_SIZE_BITS + CONFIG::NUM_STEP_BITS;
+ static constexpr size_t BITS_PER_ENTRY =
+ cpp::numeric_limits<uintptr_t>::digits;
+ static constexpr size_t TOTAL_BITS =
+ CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
+ static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_LARGE_FREELIST;
-private:
+public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
- static constexpr size_t MIN_LARGE_OUTER_SIZE = align_up(
- BlockRef::HEADER_SIZE + sizeof(FreeTrie::Node), BlockRef::MIN_ALIGN);
- static constexpr size_t NUM_SMALL_SIZES =
- (MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / BlockRef::MIN_ALIGN;
+ LIBC_INLINE TLSFFreeStoreImpl() = default;
+ LIBC_INLINE TLSFFreeStoreImpl(const TLSFFreeStoreImpl &other) = delete;
+ LIBC_INLINE TLSFFreeStoreImpl &
+ operator=(const TLSFFreeStoreImpl &other) = delete;
+
+ LIBC_INLINE void insert(BlockRef block);
+ LIBC_INLINE void remove(BlockRef block);
+ LIBC_INLINE BlockRef remove_best_fit(size_t size) {
+ return find_and_remove_fit(size);
+ }
+ LIBC_INLINE BlockRef find_and_remove_fit(size_t size);
+
+protected:
LIBC_INLINE static bool too_small(BlockRef block) {
return block.outer_size() < MIN_OUTER_SIZE;
}
- LIBC_INLINE static bool is_small(BlockRef block) {
- return block.outer_size() < MIN_LARGE_OUTER_SIZE;
- }
- FreeList &small_list(BlockRef block);
- FreeList *find_best_small_fit(size_t size);
+ union ListOrTrie {
+ FreeList list;
+ FreeTrie::Node *trie_root;
+
+ LIBC_INLINE constexpr ListOrTrie() : trie_root(nullptr) {}
+ };
- cpp::array<FreeList, NUM_SMALL_SIZES> small_lists;
- FreeTrie large_trie;
+ cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
+ cpp::array<ListOrTrie, TOTAL_BITS> free_lists{};
+
+ LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
+ LIBC_INLINE void set_bit(size_t bit_index);
+ LIBC_INLINE void clear_bit(size_t bit_index);
+ LIBC_INLINE bool get_bit(size_t bit_index) const;
+ LIBC_INLINE size_t find_first_bit_set_after(size_t bit_index) const;
+ LIBC_INLINE BlockRef remove_first_fit_in_list(size_t index, size_t size);
+ LIBC_INLINE static constexpr FreeTrie::SizeRange index_to_range(size_t index);
+ LIBC_INLINE FreeTrie get_trie(size_t index);
+ LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t index, size_t size);
+ LIBC_INLINE BlockRef pop_min_in_trie(size_t index);
};
-LIBC_INLINE void FreeStore::insert(BlockRef block) {
- if (too_small(block))
- return;
- if (is_small(block))
- small_list(block).push(block);
- else
- large_trie.push(block);
+template <typename CONFIG>
+LIBC_INLINE constexpr size_t
+TLSFFreeStoreImpl<CONFIG>::size_to_bit_index(size_t size) {
+ if (size <= (EXP_BASE << UNIT_SIZE_LOG2))
+ return size >> UNIT_SIZE_LOG2;
+
+ size_t size_ilog2 = static_cast<size_t>(cpp::bit_width(size) - 1);
+ size_t exp_offset = (size_ilog2 - UNIT_SIZE_LOG2 - EXP_BASE_LOG2 - 1)
+ << CONFIG::NUM_STEP_BITS;
+ size_t step_index = size >> (size_ilog2 - CONFIG::NUM_STEP_BITS);
+ size_t index = EXP_BASE + exp_offset + step_index;
+
+ return index < TOTAL_BITS ? index : TOTAL_BITS - 1;
}
-LIBC_INLINE void FreeStore::remove(BlockRef block) {
- if (too_small(block))
- return;
- if (is_small(block)) {
- small_list(block).remove(
- reinterpret_cast<FreeList::Node *>(block.usable_space()));
- } else {
- large_trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::set_bit(size_t bit_index) {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ lookup_table[entry_index] |= uintptr_t(1) << bit_offset;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::clear_bit(size_t bit_index) {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ lookup_table[entry_index] &= ~(uintptr_t(1) << bit_offset);
+}
+
+template <typename CONFIG>
+LIBC_INLINE bool TLSFFreeStoreImpl<CONFIG>::get_bit(size_t bit_index) const {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ return (lookup_table[entry_index] & (uintptr_t(1) << bit_offset)) != 0;
+}
+
+template <typename CONFIG>
+LIBC_INLINE size_t
+TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
+ if (bit_index >= TOTAL_BITS - 1)
+ return TOTAL_BITS;
+
+ size_t target_index = bit_index + 1;
+ size_t start_entry = target_index / BITS_PER_ENTRY;
+ size_t bit_offset = target_index % BITS_PER_ENTRY;
+
+ uintptr_t value = lookup_table[start_entry] & (~uintptr_t(0) << bit_offset);
+ if (value != 0)
+ return start_entry * BITS_PER_ENTRY +
+ static_cast<size_t>(cpp::countr_zero(value));
+
+ for (size_t i = start_entry + 1; i < CONFIG::NUM_TABLE_ENTRIES; ++i) {
+ value = lookup_table[i];
+ if (value != 0)
+ return i * BITS_PER_ENTRY + static_cast<size_t>(cpp::countr_zero(value));
}
+ return TOTAL_BITS;
}
-LIBC_INLINE BlockRef FreeStore::remove_best_fit(size_t size) {
- if (FreeList *list = find_best_small_fit(size)) {
- BlockRef block = list->front();
- list->pop();
- return block;
+template <typename CONFIG>
+LIBC_INLINE constexpr FreeTrie::SizeRange
+TLSFFreeStoreImpl<CONFIG>::index_to_range(size_t index) {
+ LIBC_ASSERT(index >= EXP_BASE && "only call for large lists");
+ size_t local_index = index - EXP_BASE;
+ size_t exp_index = local_index >> CONFIG::NUM_STEP_BITS;
+ size_t linear_index = local_index & (NUM_STEPS - 1);
+
+ size_t row_base = (EXP_BASE << exp_index) << UNIT_SIZE_LOG2;
+ size_t step_size = (STEP_SIZE << exp_index) << UNIT_SIZE_LOG2;
+ size_t min_size = row_base + linear_index * step_size;
+
+ if (index == TOTAL_BITS - 1) {
+ constexpr size_t width = size_t(1)
+ << (cpp::numeric_limits<size_t>::digits - 2);
+ LIBC_ASSERT(min_size < width &&
+ "min_size too large for overflow bin width");
+ return FreeTrie::SizeRange(min_size, width);
}
- if (FreeTrie::Node *best_fit = large_trie.find_best_fit(size)) {
+
+ return FreeTrie::SizeRange(min_size, step_size);
+}
+
+template <typename CONFIG>
+LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie(size_t index) {
+ LIBC_ASSERT(index >= EXP_BASE && "only call for large lists");
+ return FreeTrie(free_lists[index].trie_root, index_to_range(index));
+}
+
+template <typename CONFIG>
+LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit_in_trie(
+ size_t index, size_t size) {
+ FreeTrie trie = get_trie(index);
+ if (FreeTrie::Node *best_fit = trie.find_best_fit(size)) {
BlockRef block = best_fit->block();
- large_trie.remove(best_fit);
+ trie.remove(best_fit);
+ if (trie.empty())
+ clear_bit(index);
return block;
}
return BlockRef();
}
-LIBC_INLINE FreeList &FreeStore::small_list(BlockRef block) {
- LIBC_ASSERT(is_small(block) && "only legal for small blocks");
- return small_lists[(block.outer_size() - MIN_OUTER_SIZE) /
- BlockRef::MIN_ALIGN];
+template <typename CONFIG>
+LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::pop_min_in_trie(size_t index) {
+ FreeTrie trie = get_trie(index);
+ FreeTrie::Node *min_node = trie.pop_min();
+ LIBC_ASSERT(min_node && "bit was set but trie is empty");
+ BlockRef block = min_node->block();
+ if (trie.empty())
+ clear_bit(index);
+ return block;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
+ if (too_small(block))
+ return;
+ size_t bit_index = size_to_bit_index(block.inner_size());
+
+ if constexpr (USE_TRIE)
+ if (bit_index > EXP_BASE) {
+ get_trie(bit_index).push(block);
+ set_bit(bit_index);
+ return;
+ }
+
+ free_lists[bit_index].list.push(block);
+ set_bit(bit_index);
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
+ if (too_small(block))
+ return;
+ size_t bit_index = size_to_bit_index(block.inner_size());
+
+ if constexpr (USE_TRIE)
+ if (bit_index > EXP_BASE) {
+ FreeTrie trie = get_trie(bit_index);
+ trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
+ if (trie.empty())
+ clear_bit(bit_index);
+ return;
+ }
+
+ free_lists[bit_index].list.remove(
+ reinterpret_cast<FreeList::Node *>(block.usable_space()));
+ if (free_lists[bit_index].list.empty())
+ clear_bit(bit_index);
+}
+
+template <typename CONFIG>
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
+ FreeList::Node *begin_node = free_lists[index].list.begin();
+ if (begin_node == nullptr)
+ return BlockRef();
+
+ FreeList::Node *cur = begin_node;
+ do {
+ if (cur->size() >= size) {
+ free_lists[index].list.remove(cur);
+ if (free_lists[index].list.empty())
+ clear_bit(index);
+ return cur->block();
+ }
+ cur = cur->next_node();
+ } while (cur != begin_node);
+
+ return BlockRef();
}
-LIBC_INLINE FreeList *FreeStore::find_best_small_fit(size_t size) {
- for (FreeList &list : small_lists)
- if (!list.empty() && list.size() >= size)
- return &list;
- return nullptr;
+template <typename CONFIG>
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
+ size_t bit_index = size_to_bit_index(size);
+
+ if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1)) {
+ if constexpr (USE_TRIE)
+ return find_and_remove_fit_in_trie(TOTAL_BITS - 1, size);
+ else
+ return remove_first_fit_in_list(TOTAL_BITS - 1, size);
+ }
+
+ // 1. Try oversized bins (guaranteed fit, but larger).
+ size_t oversized_bit = find_first_bit_set_after(bit_index);
+ if (LIBC_LIKELY(oversized_bit < TOTAL_BITS)) {
+ if constexpr (USE_TRIE)
+ if (oversized_bit > EXP_BASE)
+ return pop_min_in_trie(oversized_bit);
+
+ BlockRef block = free_lists[oversized_bit].list.front();
+ free_lists[oversized_bit].list.pop();
+ if (free_lists[oversized_bit].list.empty())
+ clear_bit(oversized_bit);
+ return block;
+ }
+
+ // 2. Try exact fit (fallback).
+ if (get_bit(bit_index)) {
+ if constexpr (USE_TRIE) {
+ if (bit_index > EXP_BASE)
+ return find_and_remove_fit_in_trie(bit_index, size);
+ else if (BlockRef block = remove_first_fit_in_list(bit_index, size))
+ return block;
+ } else if (BlockRef block = remove_first_fit_in_list(bit_index, size))
+ return block;
+ }
+
+ return BlockRef();
}
+template <size_t UNIT_SIZE, size_t STEP_SIZE_BITS, size_t NUM_STEP_BITS,
+ size_t NUM_TABLE_ENTRIES, bool USE_TRIE = false>
+using TLSFFreeStore = TLSFFreeStoreImpl<TLSFFreeStoreConfig<
+ UNIT_SIZE, STEP_SIZE_BITS, NUM_STEP_BITS, NUM_TABLE_ENTRIES, USE_TRIE>>;
+
+#ifndef LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST
+#define LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST false
+#endif
+
+using FreeStore =
+ TLSFFreeStore<BlockRef::MIN_ALIGN, 3, 2, (sizeof(uintptr_t) == 8 ? 3 : 6),
+ LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST>;
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 9e35463462b38..9e07f36ad0a53 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -87,15 +87,8 @@ class FreeTrie {
}
};
- LIBC_INLINE constexpr FreeTrie() : FreeTrie(SizeRange{0, 0}) {}
- LIBC_INLINE constexpr FreeTrie(SizeRange range) : range(range) {}
-
- /// Sets the range of possible block sizes. This can only be called when the
- /// trie is empty.
- LIBC_INLINE void set_range(FreeTrie::SizeRange new_range) {
- LIBC_ASSERT(empty() && "cannot change the range of a preexisting trie");
- range = new_range;
- }
+ LIBC_INLINE FreeTrie(Node *&root, SizeRange range)
+ : root(root), range(range) {}
/// @returns Whether the trie contains any blocks.
LIBC_INLINE bool empty() const { return !root; }
@@ -110,15 +103,24 @@ class FreeTrie {
/// nullptr.
Node *find_best_fit(size_t size);
+ /// @returns The node with the minimum size in the trie; otherwise nullptr.
+ LIBC_INLINE Node *find_min();
+
+ /// Removes and returns the node with the minimum size in the trie; otherwise
+ /// nullptr.
+ LIBC_INLINE Node *pop_min();
+
private:
/// @returns Whether a node is the head of its containing freelist.
- bool is_head(Node *node) const { return node->parent || node == root; }
+ LIBC_INLINE bool is_head(Node *node) const {
+ return node->parent || node == root;
+ }
/// Replaces references to one node with another (or nullptr) in all adjacent
/// parent and child nodes.
void replace_node(Node *node, Node *new_node);
- Node *root = nullptr;
+ Node *&root;
SizeRange range;
};
@@ -236,6 +238,47 @@ LIBC_INLINE FreeTrie::Node *FreeTrie::find_best_fit(size_t size) {
}
}
}
+LIBC_INLINE FreeTrie::Node *FreeTrie::find_min() {
+ if (empty())
+ return nullptr;
+
+ Node *cur = root;
+ SizeRange cur_range = range;
+ Node *best_min = nullptr;
+
+ while (cur) {
+ if (cur->lower) {
+ if (cur_range.lower().contains(cur->size())) {
+ if (!best_min || cur->size() < best_min->size()) {
+ best_min = cur;
+ }
+ }
+ cur = cur->lower;
+ cur_range = cur_range.lower();
+ } else {
+ if (cur_range.lower().contains(cur->size())) {
+ if (!best_min || cur->size() < best_min->size()) {
+ best_min = cur;
+ }
+ break;
+ } else {
+ if (!best_min || cur->size() < best_min->size()) {
+ best_min = cur;
+ }
+ cur = cur->upper;
+ cur_range = cur_range.upper();
+ }
+ }
+ }
+ return best_min;
+}
+
+LIBC_INLINE FreeTrie::Node *FreeTrie::pop_min() {
+ Node *min_node = find_min();
+ if (min_node)
+ remove(min_node);
+ return min_node;
+}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index 61103a9126c08..ce655367da00f 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -38,7 +38,6 @@ TEST(LlvmLibcFreeStore, TooSmall) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(too_small);
store.insert(remainder);
@@ -68,21 +67,26 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(smallest);
if (largest_small != smallest)
store.insert(largest_small);
store.insert(remainder);
- // Find exact match for smallest.
- ASSERT_EQ(store.remove_best_fit(smallest.inner_size()).addr(),
- smallest.addr());
- store.insert(smallest);
-
- // Find exact match for largest.
- ASSERT_EQ(store.remove_best_fit(largest_small.inner_size()).addr(),
- largest_small.addr());
- store.insert(largest_small);
+ // For TLSF (oversized first), asking for a size will return the block from
+ // the first non-empty oversized bin if one exists, bypassing the exact bin.
+ if (largest_small != smallest) {
+ BlockRef block = store.remove_best_fit(smallest.inner_size());
+ ASSERT_EQ(block.addr(), largest_small.addr());
+ store.insert(block);
+
+ BlockRef block2 = store.remove_best_fit(largest_small.inner_size());
+ ASSERT_EQ(block2.addr(), remainder.addr());
+ store.insert(block2);
+ } else {
+ BlockRef block = store.remove_best_fit(smallest.inner_size());
+ ASSERT_EQ(block.addr(), remainder.addr());
+ store.insert(block);
+ }
// Search small list for best fit.
BlockRef next_smallest =
@@ -108,7 +112,6 @@ TEST(LlvmLibcFreeStore, Remove) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(small);
store.insert(remainder);
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index bf3284b2faf64..7eea72bcdbc37 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -22,7 +22,9 @@ using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::optional;
TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
- FreeTrie trie({0, 4096});
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
EXPECT_EQ(trie.find_best_fit(123), static_cast<FreeTrie::Node *>(nullptr));
byte mem[1024];
@@ -31,10 +33,10 @@ TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
BlockRef block = *maybeBlock;
trie.push(block);
- FreeTrie::Node *root = trie.find_best_fit(0);
- ASSERT_EQ(root->block().addr(), block.addr());
- EXPECT_EQ(trie.find_best_fit(block.inner_size() - 1), root);
- EXPECT_EQ(trie.find_best_fit(block.inner_size()), root);
+ FreeTrie::Node *found = trie.find_best_fit(0);
+ ASSERT_EQ(found->block().addr(), block.addr());
+ EXPECT_EQ(trie.find_best_fit(block.inner_size() - 1), found);
+ EXPECT_EQ(trie.find_best_fit(block.inner_size()), found);
EXPECT_EQ(trie.find_best_fit(block.inner_size() + 1),
static_cast<FreeTrie::Node *>(nullptr));
EXPECT_EQ(trie.find_best_fit(4095), static_cast<FreeTrie::Node *>(nullptr));
@@ -47,10 +49,12 @@ TEST(LlvmLibcFreeTrie, FindBestFitLower) {
BlockRef lower = *maybeBlock;
maybeBlock = lower.split(512);
ASSERT_TRUE(maybeBlock.has_value());
- BlockRef root = *maybeBlock;
+ BlockRef root_block = *maybeBlock;
- FreeTrie trie({0, 4096});
- trie.push(root);
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
+ trie.push(root_block);
trie.push(lower);
EXPECT_EQ(trie.find_best_fit(0)->block().addr(), lower.addr());
@@ -60,36 +64,40 @@ TEST(LlvmLibcFreeTrie, FindBestFitUpper) {
byte mem[4096];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
- BlockRef root = *maybeBlock;
- maybeBlock = root.split(512);
+ BlockRef root_block = *maybeBlock;
+ maybeBlock = root_block.split(512);
ASSERT_TRUE(maybeBlock.has_value());
BlockRef upper = *maybeBlock;
- FreeTrie trie({0, 4096});
- trie.push(root);
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
+ trie.push(root_block);
trie.push(upper);
- EXPECT_EQ(trie.find_best_fit(root.inner_size() + 1)->block().addr(),
+ EXPECT_EQ(trie.find_best_fit(root_block.inner_size() + 1)->block().addr(),
upper.addr());
// The upper subtrie should be skipped if it could not contain a better fit.
- EXPECT_EQ(trie.find_best_fit(root.inner_size() - 1)->block().addr(),
- root.addr());
+ EXPECT_EQ(trie.find_best_fit(root_block.inner_size() - 1)->block().addr(),
+ root_block.addr());
}
TEST(LlvmLibcFreeTrie, FindBestFitLowerAndUpper) {
byte mem[4096];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
- BlockRef root = *maybeBlock;
- maybeBlock = root.split(1024);
+ BlockRef root_block = *maybeBlock;
+ maybeBlock = root_block.split(1024);
ASSERT_TRUE(maybeBlock.has_value());
BlockRef lower = *maybeBlock;
maybeBlock = lower.split(128);
ASSERT_TRUE(maybeBlock.has_value());
BlockRef upper = *maybeBlock;
- FreeTrie trie({0, 4096});
- trie.push(root);
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
+ trie.push(root_block);
trie.push(lower);
trie.push(upper);
@@ -114,7 +122,9 @@ TEST(LlvmLibcFreeTrie, Remove) {
BlockRef large = *maybeBlock;
// Removing the root empties the trie.
- FreeTrie trie({0, 4096});
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
trie.push(large);
FreeTrie::Node *large_node = trie.find_best_fit(0);
ASSERT_EQ(large_node->block().addr(), large.addr());
@@ -132,3 +142,46 @@ TEST(LlvmLibcFreeTrie, Remove) {
EXPECT_EQ(trie.find_best_fit(large.inner_size())->block().addr(),
large.addr());
}
+
+TEST(LlvmLibcFreeTrie, PopMin) {
+ alignas(BlockRef::MIN_ALIGN) byte mem[4096];
+ optional<BlockRef> maybe_block = BlockRef::init(mem);
+ ASSERT_TRUE(maybe_block.has_value());
+ BlockRef root_block = *maybe_block;
+ maybe_block = root_block.split(1024);
+ ASSERT_TRUE(maybe_block.has_value());
+ BlockRef lower = *maybe_block;
+ maybe_block = lower.split(128);
+ ASSERT_TRUE(maybe_block.has_value());
+ BlockRef upper = *maybe_block;
+
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
+
+ // Empty pop
+ EXPECT_EQ(trie.pop_min(), static_cast<FreeTrie::Node *>(nullptr));
+
+ trie.push(root_block);
+ trie.push(lower);
+ trie.push(upper);
+
+ // Min should be lower (~128)
+ FreeTrie::Node *min1 = trie.pop_min();
+ ASSERT_NE(min1, static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_EQ(min1->block().addr(), lower.addr());
+
+ // Next min should be root_block (~1024)
+ FreeTrie::Node *min2 = trie.pop_min();
+ ASSERT_NE(min2, static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_EQ(min2->block().addr(), root_block.addr());
+
+ // Next min should be upper (~2944)
+ FreeTrie::Node *min3 = trie.pop_min();
+ ASSERT_NE(min3, static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_EQ(min3->block().addr(), upper.addr());
+
+ // Now empty
+ EXPECT_EQ(trie.pop_min(), static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_TRUE(trie.empty());
+}
>From a193a91eb59cad755f51d35e4672bea893f136c0 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Thu, 11 Jun 2026 15:23:40 -0700
Subject: [PATCH 2/3] [libc] restrict FreeTrie to the overflow bin in TLSF
- Rename config option to USE_TRIE_FOR_OVERFLOW_BIN.
- Restrict FreeTrie usage to only the last overflow bin (bit_index == TOTAL_BITS - 1).
- Simplify exact fit search to only use FreeList, as large sizes now always go to the overflow bin.
- Simplify away index_to_range by using a fixed [0, INF) range for the overflow Trie.
- Remove index argument from Trie helper functions as they only operate on the overflow bin.
TAG=agy
CONV=fe3b4efa-7a5b-4c74-8257-e53f0d6e4850
---
libc/src/__support/freestore.h | 90 ++++++++++++----------------------
1 file changed, 32 insertions(+), 58 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 4fb90cabf47bd..13b79c0debf39 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -30,14 +30,14 @@ namespace LIBC_NAMESPACE_DECL {
/// Configuration for TLSFFreeStore.
template <size_t UNIT_SIZE_VAL, size_t STEP_SIZE_BITS_VAL,
size_t NUM_STEP_BITS_VAL, size_t NUM_TABLE_ENTRIES_VAL,
- bool USE_TRIE_FOR_LARGE_FREELIST_VAL = false>
+ bool USE_TRIE_FOR_OVERFLOW_BIN_VAL = false>
struct TLSFFreeStoreConfig {
static constexpr size_t UNIT_SIZE = UNIT_SIZE_VAL;
static constexpr size_t STEP_SIZE_BITS = STEP_SIZE_BITS_VAL;
static constexpr size_t NUM_STEP_BITS = NUM_STEP_BITS_VAL;
static constexpr size_t NUM_TABLE_ENTRIES = NUM_TABLE_ENTRIES_VAL;
- static constexpr bool USE_TRIE_FOR_LARGE_FREELIST =
- USE_TRIE_FOR_LARGE_FREELIST_VAL;
+ static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN =
+ USE_TRIE_FOR_OVERFLOW_BIN_VAL;
};
// A two-level segregated fit store for free blocks.
@@ -108,7 +108,9 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
cpp::numeric_limits<uintptr_t>::digits;
static constexpr size_t TOTAL_BITS =
CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
- static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_LARGE_FREELIST;
+ static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_OVERFLOW_BIN;
+ static constexpr size_t OVERFLOW_WIDTH =
+ size_t(1) << (cpp::numeric_limits<size_t>::digits - 2);
public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
@@ -147,10 +149,9 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
LIBC_INLINE bool get_bit(size_t bit_index) const;
LIBC_INLINE size_t find_first_bit_set_after(size_t bit_index) const;
LIBC_INLINE BlockRef remove_first_fit_in_list(size_t index, size_t size);
- LIBC_INLINE static constexpr FreeTrie::SizeRange index_to_range(size_t index);
- LIBC_INLINE FreeTrie get_trie(size_t index);
- LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t index, size_t size);
- LIBC_INLINE BlockRef pop_min_in_trie(size_t index);
+ LIBC_INLINE FreeTrie get_trie();
+ LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t size);
+ LIBC_INLINE BlockRef pop_min_in_trie();
};
template <typename CONFIG>
@@ -213,56 +214,33 @@ TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
}
template <typename CONFIG>
-LIBC_INLINE constexpr FreeTrie::SizeRange
-TLSFFreeStoreImpl<CONFIG>::index_to_range(size_t index) {
- LIBC_ASSERT(index >= EXP_BASE && "only call for large lists");
- size_t local_index = index - EXP_BASE;
- size_t exp_index = local_index >> CONFIG::NUM_STEP_BITS;
- size_t linear_index = local_index & (NUM_STEPS - 1);
-
- size_t row_base = (EXP_BASE << exp_index) << UNIT_SIZE_LOG2;
- size_t step_size = (STEP_SIZE << exp_index) << UNIT_SIZE_LOG2;
- size_t min_size = row_base + linear_index * step_size;
-
- if (index == TOTAL_BITS - 1) {
- constexpr size_t width = size_t(1)
- << (cpp::numeric_limits<size_t>::digits - 2);
- LIBC_ASSERT(min_size < width &&
- "min_size too large for overflow bin width");
- return FreeTrie::SizeRange(min_size, width);
- }
-
- return FreeTrie::SizeRange(min_size, step_size);
-}
-
-template <typename CONFIG>
-LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie(size_t index) {
- LIBC_ASSERT(index >= EXP_BASE && "only call for large lists");
- return FreeTrie(free_lists[index].trie_root, index_to_range(index));
+LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
+ return FreeTrie(free_lists[TOTAL_BITS - 1].trie_root,
+ FreeTrie::SizeRange(0, OVERFLOW_WIDTH));
}
template <typename CONFIG>
-LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit_in_trie(
- size_t index, size_t size) {
- FreeTrie trie = get_trie(index);
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit_in_trie(size_t size) {
+ FreeTrie trie = get_trie();
if (FreeTrie::Node *best_fit = trie.find_best_fit(size)) {
BlockRef block = best_fit->block();
trie.remove(best_fit);
if (trie.empty())
- clear_bit(index);
+ clear_bit(TOTAL_BITS - 1);
return block;
}
return BlockRef();
}
template <typename CONFIG>
-LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::pop_min_in_trie(size_t index) {
- FreeTrie trie = get_trie(index);
+LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::pop_min_in_trie() {
+ FreeTrie trie = get_trie();
FreeTrie::Node *min_node = trie.pop_min();
LIBC_ASSERT(min_node && "bit was set but trie is empty");
BlockRef block = min_node->block();
if (trie.empty())
- clear_bit(index);
+ clear_bit(TOTAL_BITS - 1);
return block;
}
@@ -273,8 +251,8 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
size_t bit_index = size_to_bit_index(block.inner_size());
if constexpr (USE_TRIE)
- if (bit_index > EXP_BASE) {
- get_trie(bit_index).push(block);
+ if (bit_index == TOTAL_BITS - 1) {
+ get_trie().push(block);
set_bit(bit_index);
return;
}
@@ -290,8 +268,8 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
size_t bit_index = size_to_bit_index(block.inner_size());
if constexpr (USE_TRIE)
- if (bit_index > EXP_BASE) {
- FreeTrie trie = get_trie(bit_index);
+ if (bit_index == TOTAL_BITS - 1) {
+ FreeTrie trie = get_trie();
trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
if (trie.empty())
clear_bit(bit_index);
@@ -332,7 +310,7 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1)) {
if constexpr (USE_TRIE)
- return find_and_remove_fit_in_trie(TOTAL_BITS - 1, size);
+ return find_and_remove_fit_in_trie(size);
else
return remove_first_fit_in_list(TOTAL_BITS - 1, size);
}
@@ -340,9 +318,10 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
// 1. Try oversized bins (guaranteed fit, but larger).
size_t oversized_bit = find_first_bit_set_after(bit_index);
if (LIBC_LIKELY(oversized_bit < TOTAL_BITS)) {
- if constexpr (USE_TRIE)
- if (oversized_bit > EXP_BASE)
- return pop_min_in_trie(oversized_bit);
+ if constexpr (USE_TRIE) {
+ if (oversized_bit == TOTAL_BITS - 1)
+ return pop_min_in_trie();
+ }
BlockRef block = free_lists[oversized_bit].list.front();
free_lists[oversized_bit].list.pop();
@@ -353,12 +332,7 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
// 2. Try exact fit (fallback).
if (get_bit(bit_index)) {
- if constexpr (USE_TRIE) {
- if (bit_index > EXP_BASE)
- return find_and_remove_fit_in_trie(bit_index, size);
- else if (BlockRef block = remove_first_fit_in_list(bit_index, size))
- return block;
- } else if (BlockRef block = remove_first_fit_in_list(bit_index, size))
+ if (BlockRef block = remove_first_fit_in_list(bit_index, size))
return block;
}
@@ -370,13 +344,13 @@ template <size_t UNIT_SIZE, size_t STEP_SIZE_BITS, size_t NUM_STEP_BITS,
using TLSFFreeStore = TLSFFreeStoreImpl<TLSFFreeStoreConfig<
UNIT_SIZE, STEP_SIZE_BITS, NUM_STEP_BITS, NUM_TABLE_ENTRIES, USE_TRIE>>;
-#ifndef LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST
-#define LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST false
+#ifndef LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN
+#define LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN false
#endif
using FreeStore =
TLSFFreeStore<BlockRef::MIN_ALIGN, 3, 2, (sizeof(uintptr_t) == 8 ? 3 : 6),
- LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST>;
+ LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN>;
} // namespace LIBC_NAMESPACE_DECL
>From 7e4b51dbaaf4be78dad3987e9ea4835ad1c562ff Mon Sep 17 00:00:00 2001
From: yfzhu <yfzhu at google.com>
Date: Tue, 23 Jun 2026 09:15:24 -0700
Subject: [PATCH 3/3] [libc] make baremetal freelist heap headers-only
This change moves the implementations of FreeList, FreeTrie and FreeListHeap
to their respective header files, making the baremetal freelist heap library
headers-only. This simplifies integration and avoids compilation issues.
TAG=agy
CONV=4118a58d-30c6-4fd4-8e45-defe707d8bba
---
libc/src/__support/CMakeLists.txt | 15 +--
libc/src/__support/freelist.cpp | 47 ----------
libc/src/__support/freelist.h | 31 ++++++-
libc/src/__support/freelist_heap.cpp | 19 ----
libc/src/__support/freelist_heap.h | 26 ++++--
libc/src/__support/freetrie.cpp | 64 -------------
libc/src/__support/freetrie.h | 56 ++++++++++-
.../llvm-project-overlay/libc/BUILD.bazel | 92 +++++++++++++++++++
8 files changed, 194 insertions(+), 156 deletions(-)
delete mode 100644 libc/src/__support/freelist.cpp
delete mode 100644 libc/src/__support/freelist_heap.cpp
delete mode 100644 libc/src/__support/freetrie.cpp
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 9d08b4bcf7303..2c5f38a8d2f72 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -25,12 +25,10 @@ add_header_library(
libc.src.__support.math_extras
)
-add_object_library(
+add_header_library(
freelist
HDRS
freelist.h
- SRCS
- freelist.cpp
DEPENDS
.block
libc.src.__support.fixedvector
@@ -40,12 +38,10 @@ add_object_library(
libc.src.__support.CPP.span
)
-add_object_library(
+add_header_library(
freetrie
HDRS
freetrie.h
- SRCS
- freetrie.cpp
DEPENDS
.block
.freelist
@@ -59,15 +55,10 @@ add_header_library(
.freetrie
)
-libc_set_definition(libc_freelist_malloc_size "LIBC_FREELIST_MALLOC_SIZE=${LIBC_CONF_FREELIST_MALLOC_BUFFER_SIZE}")
-add_object_library(
+add_header_library(
freelist_heap
- SRCS
- freelist_heap.cpp
HDRS
freelist_heap.h
- COMPILE_OPTIONS
- ${libc_freelist_malloc_size}
DEPENDS
.block
.freelist
diff --git a/libc/src/__support/freelist.cpp b/libc/src/__support/freelist.cpp
deleted file mode 100644
index 28f73b2726d7d..0000000000000
--- a/libc/src/__support/freelist.cpp
+++ /dev/null
@@ -1,47 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// Implementation for freelist.
-///
-//===----------------------------------------------------------------------===//
-
-#include "freelist.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-void FreeList::push(Node *node) {
- if (begin_) {
- LIBC_ASSERT(BlockRef::from_usable_space(node).outer_size() ==
- begin_->block().outer_size() &&
- "freelist entries must have the same size");
- // Since the list is circular, insert the node immediately before begin_.
- node->prev = begin_->prev;
- node->next = begin_;
- begin_->prev->next = node;
- begin_->prev = node;
- } else {
- begin_ = node->prev = node->next = node;
- }
-}
-
-void FreeList::remove(Node *node) {
- LIBC_ASSERT(begin_ && "cannot remove from empty list");
- if (node == node->next) {
- LIBC_ASSERT(node == begin_ &&
- "a self-referential node must be the only element");
- begin_ = nullptr;
- } else {
- node->prev->next = node->next;
- node->next->prev = node->prev;
- if (begin_ == node)
- begin_ = node->next;
- }
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index ae2de684b3a24..34d8f759fd0bb 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -15,6 +15,8 @@
#define LLVM_LIBC_SRC___SUPPORT_FREELIST_H
#include "block.h"
+#include "src/__support/libc_assert.h"
+#include "src/__support/macros/config.h"
namespace LIBC_NAMESPACE_DECL {
@@ -80,13 +82,38 @@ class FreeList {
/// Push an already-constructed node to the back of the list.
/// This allows pushing derived node types with additional data.
- void push(Node *node);
+ LIBC_INLINE void push(Node *node) {
+ if (begin_) {
+ LIBC_ASSERT(BlockRef::from_usable_space(node).outer_size() ==
+ begin_->block().outer_size() &&
+ "freelist entries must have the same size");
+ // Since the list is circular, insert the node immediately before begin_.
+ node->prev = begin_->prev;
+ node->next = begin_;
+ begin_->prev->next = node;
+ begin_->prev = node;
+ } else {
+ begin_ = node->prev = node->next = node;
+ }
+ }
/// Pop the first node from the list.
LIBC_INLINE void pop() { remove(begin_); }
/// Remove an arbitrary node from the list.
- void remove(Node *node);
+ LIBC_INLINE void remove(Node *node) {
+ LIBC_ASSERT(begin_ && "cannot remove from empty list");
+ if (node == node->next) {
+ LIBC_ASSERT(node == begin_ &&
+ "a self-referential node must be the only element");
+ begin_ = nullptr;
+ } else {
+ node->prev->next = node->next;
+ node->next->prev = node->prev;
+ if (begin_ == node)
+ begin_ = node->next;
+ }
+ }
private:
Node *begin_;
diff --git a/libc/src/__support/freelist_heap.cpp b/libc/src/__support/freelist_heap.cpp
deleted file mode 100644
index 4deb0e0f09e22..0000000000000
--- a/libc/src/__support/freelist_heap.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation for freelist_heap ----------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/__support/freelist_heap.h"
-#include "src/__support/macros/config.h"
-
-#include <stddef.h>
-
-namespace LIBC_NAMESPACE_DECL {
-
-static LIBC_CONSTINIT FreeListHeap freelist_heap_symbols;
-FreeListHeap *freelist_heap = &freelist_heap_symbols;
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 976a6eec4023e..c05bf806990c8 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -21,6 +21,7 @@
#include "src/__support/CPP/optional.h"
#include "src/__support/CPP/span.h"
#include "src/__support/libc_assert.h"
+#include "src/__support/macros/attributes.h"
#include "src/__support/macros/config.h"
#include "src/__support/math_extras.h"
#include "src/string/memory_utils/inline_memcpy.h"
@@ -80,7 +81,7 @@ template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
cpp::byte buffer[BUFF_SIZE];
};
-LIBC_INLINE void FreeListHeap::init() {
+[[gnu::noinline]] LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
@@ -88,7 +89,8 @@ LIBC_INLINE void FreeListHeap::init() {
is_initialized = true;
}
-LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *
+FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (size == 0)
return nullptr;
@@ -113,12 +115,12 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
return block_info.block.usable_space();
}
-LIBC_INLINE void *FreeListHeap::allocate(size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *FreeListHeap::allocate(size_t size) {
return allocate_impl(BlockRef::MIN_ALIGN, size);
}
-LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment,
- size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *
+FreeListHeap::aligned_allocate(size_t alignment, size_t size) {
// The alignment must be an integral power of two.
if (!IsPow2(alignment))
return nullptr;
@@ -133,7 +135,7 @@ LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment,
return allocate_impl(alignment, size);
}
-LIBC_INLINE void FreeListHeap::free(void *ptr) {
+[[gnu::noinline]] LIBC_INLINE void FreeListHeap::free(void *ptr) {
if (ptr == nullptr)
return;
@@ -164,7 +166,8 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
free_store.insert(block);
}
-LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
+[[gnu::noinline]] LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block,
+ size_t size) {
size_t min_outer_size = BlockRef::outer_size(cpp::max(size, sizeof(size_t)));
uintptr_t next_block_start = BlockRef::next_possible_block_start(
block.addr() + min_outer_size, BlockRef::MIN_ALIGN);
@@ -193,7 +196,8 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
// Follows constract of the C standard realloc() function
// If ptr is free'd, will return nullptr.
-LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *FreeListHeap::realloc(void *ptr,
+ size_t size) {
if (size == 0) {
free(ptr);
return nullptr;
@@ -228,7 +232,8 @@ LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
return new_ptr;
}
-LIBC_INLINE void *FreeListHeap::calloc(size_t num, size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *FreeListHeap::calloc(size_t num,
+ size_t size) {
size_t bytes;
if (__builtin_mul_overflow(num, size, &bytes))
return nullptr;
@@ -238,7 +243,8 @@ LIBC_INLINE void *FreeListHeap::calloc(size_t num, size_t size) {
return ptr;
}
-extern FreeListHeap *freelist_heap;
+LIBC_INLINE_VAR LIBC_CONSTINIT FreeListHeap freelist_heap_symbols;
+LIBC_INLINE_VAR FreeListHeap *freelist_heap = &freelist_heap_symbols;
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freetrie.cpp b/libc/src/__support/freetrie.cpp
deleted file mode 100644
index e76efe717f215..0000000000000
--- a/libc/src/__support/freetrie.cpp
+++ /dev/null
@@ -1,64 +0,0 @@
-//===-- Implementation for freetrie ---------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "freetrie.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-void FreeTrie::remove(Node *node) {
- LIBC_ASSERT(!empty() && "cannot remove from empty trie");
- FreeList list = node;
- list.pop();
- Node *new_node = static_cast<Node *>(list.begin());
- if (!new_node) {
- // The freelist is empty. Replace the subtrie root with an arbitrary leaf.
- // This is legal because there is no relationship between the size of the
- // root and its children.
- Node *leaf = node;
- while (leaf->lower || leaf->upper)
- leaf = leaf->lower ? leaf->lower : leaf->upper;
- if (leaf == node) {
- // If the root is a leaf, then removing it empties the subtrie.
- replace_node(node, nullptr);
- return;
- }
-
- replace_node(leaf, nullptr);
- new_node = leaf;
- }
-
- if (!is_head(node))
- return;
-
- // Copy the trie links to the new head.
- new_node->lower = node->lower;
- new_node->upper = node->upper;
- new_node->parent = node->parent;
- replace_node(node, new_node);
-}
-
-void FreeTrie::replace_node(Node *node, Node *new_node) {
- LIBC_ASSERT(is_head(node) && "only head nodes contain trie links");
-
- if (node->parent) {
- Node *&parent_child =
- node->parent->lower == node ? node->parent->lower : node->parent->upper;
- LIBC_ASSERT(parent_child == node &&
- "no reference to child node found in parent");
- parent_child = new_node;
- } else {
- LIBC_ASSERT(root == node && "non-root node had no parent");
- root = new_node;
- }
- if (node->lower)
- node->lower->parent = new_node;
- if (node->upper)
- node->upper->parent = new_node;
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 9e07f36ad0a53..6c1b479ba8813 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -15,6 +15,7 @@
#define LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
#include "freelist.h"
+#include "src/__support/libc_assert.h"
namespace LIBC_NAMESPACE_DECL {
@@ -97,7 +98,7 @@ class FreeTrie {
void push(BlockRef block);
/// Remove a node from this trie node's free list.
- void remove(Node *node);
+ LIBC_INLINE void remove(Node *node);
/// @returns A smallest node that can allocate the given size; otherwise
/// nullptr.
@@ -118,7 +119,7 @@ class FreeTrie {
/// Replaces references to one node with another (or nullptr) in all adjacent
/// parent and child nodes.
- void replace_node(Node *node, Node *new_node);
+ LIBC_INLINE void replace_node(Node *node, Node *new_node);
Node *&root;
SizeRange range;
@@ -280,6 +281,57 @@ LIBC_INLINE FreeTrie::Node *FreeTrie::pop_min() {
return min_node;
}
+LIBC_INLINE void FreeTrie::remove(Node *node) {
+ LIBC_ASSERT(!empty() && "cannot remove from empty trie");
+ FreeList list = node;
+ list.pop();
+ Node *new_node = static_cast<Node *>(list.begin());
+ if (!new_node) {
+ // The freelist is empty. Replace the subtrie root with an arbitrary leaf.
+ // This is legal because there is no relationship between the size of the
+ // root and its children.
+ Node *leaf = node;
+ while (leaf->lower || leaf->upper)
+ leaf = leaf->lower ? leaf->lower : leaf->upper;
+ if (leaf == node) {
+ // If the root is a leaf, then removing it empties the subtrie.
+ replace_node(node, nullptr);
+ return;
+ }
+
+ replace_node(leaf, nullptr);
+ new_node = leaf;
+ }
+
+ if (!is_head(node))
+ return;
+
+ // Copy the trie links to the new head.
+ new_node->lower = node->lower;
+ new_node->upper = node->upper;
+ new_node->parent = node->parent;
+ replace_node(node, new_node);
+}
+
+LIBC_INLINE void FreeTrie::replace_node(Node *node, Node *new_node) {
+ LIBC_ASSERT(is_head(node) && "only head nodes contain trie links");
+
+ if (node->parent) {
+ Node *&parent_child =
+ node->parent->lower == node ? node->parent->lower : node->parent->upper;
+ LIBC_ASSERT(parent_child == node &&
+ "no reference to child node found in parent");
+ parent_child = new_node;
+ } else {
+ LIBC_ASSERT(root == node && "non-root node had no parent");
+ root = new_node;
+ }
+ if (node->lower)
+ node->lower->parent = new_node;
+ if (node->upper)
+ node->upper->parent = new_node;
+}
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 7c0a9faff678d..7f9d7f6cbac5a 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -18,6 +18,7 @@ load(
"libc_support_library",
)
load(":platforms.bzl", "PLATFORM_CPU_ARM64", "PLATFORM_CPU_X86_64")
+load(":libc_namespace.bzl", "LIBC_NAMESPACE")
package(
default_visibility = ["//visibility:public"],
@@ -16250,3 +16251,94 @@ libc_function(
":types_wchar_t",
],
)
+
+libc_support_library(
+ name = "__support_block",
+ hdrs = ["src/__support/block.h"],
+ deps = [
+ ":__support_common",
+ ":__support_cpp_algorithm",
+ ":__support_cpp_cstddef",
+ ":__support_cpp_limits",
+ ":__support_cpp_new",
+ ":__support_cpp_optional",
+ ":__support_cpp_span",
+ ":__support_libc_assert",
+ ":__support_macros_attributes",
+ ":__support_macros_config",
+ ":__support_math_extras",
+ ":hdr_stdint_proxy",
+ ":string_memory_utils",
+ ":types_size_t",
+ ],
+)
+
+libc_support_library(
+ name = "freelist",
+ hdrs = ["src/__support/freelist.h"],
+ deps = [
+ ":__support_block",
+ ":__support_common",
+ ":__support_libc_assert",
+ ":__support_macros_config",
+ ],
+)
+
+libc_support_library(
+ name = "freetrie",
+ hdrs = ["src/__support/freetrie.h"],
+ deps = [
+ ":freelist",
+ ":__support_common",
+ ":__support_libc_assert",
+ ":__support_macros_config",
+ ],
+)
+
+libc_support_library(
+ name = "freestore",
+ hdrs = ["src/__support/freestore.h"],
+ deps = [
+ ":__support_block",
+ ":__support_common",
+ ":__support_cpp_array",
+ ":__support_cpp_bit",
+ ":__support_cpp_limits",
+ ":freelist",
+ ":freetrie",
+ ":__support_macros_config",
+ ":__support_macros_optimization",
+ ":hdr_stdint_proxy",
+ ":types_size_t",
+ ],
+)
+
+libc_support_library(
+ name = "freelist_heap",
+ hdrs = ["src/__support/freelist_heap.h"],
+ deps = [
+ ":__support_block",
+ ":freestore",
+ ":__support_common",
+ ":__support_cpp_optional",
+ ":__support_cpp_span",
+ ":__support_libc_assert",
+ ":__support_macros_config",
+ ":__support_macros_attributes",
+ ":__support_math_extras",
+ ":string_memory_utils",
+ ],
+)
+
+cc_library(
+ name = "freelist_heap_lib",
+ visibility = ["//visibility:public"],
+ includes = ["."],
+ defines = [
+ "LLVM_LIBC_SRC___SUPPORT_CPP_NEW_H",
+ "LIBC_NAMESPACE=" + LIBC_NAMESPACE,
+ ],
+ deps = [
+ ":freelist_heap",
+ ],
+)
More information about the libc-commits
mailing list