[libc-commits] [libc] [llvm] [libc] add option for hardened freelist (PR #205382)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Tue Jun 23 09:59:27 PDT 2026
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/205382
>From e7eed7b4e0600b9d62c6c44ebf355eb1bbfc2c3c Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 10 Jun 2026 13:50:35 -0700
Subject: [PATCH 1/4] Refactor FreeTrie and Implement TLSF FreeStore
- Make FreeTrie a proxy holding root by reference.
- Move root and range storage outside FreeTrie (into FreeStore).
- Add find_min and pop_min implementations.
- Update tests and add pop_min test.
- Replace freestore.h with TLSF implementation.
- Add USE_TRIE_FOR_LARGE_FREELIST option to use FreeTrie in TLSF.
- Remove obsolete set_range interface from FreeStore, FreeListHeap, and tests.
TAG=agy
CONV=fe3b4efa-7a5b-4c74-8257-e53f0d6e4850
---
libc/src/__support/freelist.h | 3 +
libc/src/__support/freelist_heap.h | 1 -
libc/src/__support/freestore.h | 403 +++++++++++++++++----
libc/src/__support/freetrie.h | 65 +++-
libc/test/src/__support/freestore_test.cpp | 27 +-
libc/test/src/__support/freetrie_test.cpp | 93 ++++-
6 files changed, 478 insertions(+), 114 deletions(-)
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index 48e70c7c29df6..ae2de684b3a24 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -41,6 +41,9 @@ class FreeList {
/// @returns The inner size of blocks in the list containing this node.
LIBC_INLINE size_t size() const { return block().inner_size(); }
+ /// @returns The next node in the list containing this node.
+ LIBC_INLINE Node *next_node() const { return next; }
+
private:
// Circularly linked pointers to adjacent nodes.
Node *prev;
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index a591d63ddd4e5..976a6eec4023e 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -84,7 +84,6 @@ LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
- free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
free_store.insert(block);
is_initialized = true;
}
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index adc0e061ace93..4fb90cabf47bd 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -7,114 +7,377 @@
//===----------------------------------------------------------------------===//
///
/// \file
-/// Interface for freestore.
+/// This file contains a two-level segregated fit free block store.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
#define LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
-#include "freetrie.h"
+#include "hdr/stdint_proxy.h"
+#include "hdr/types/size_t.h"
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/block.h"
+#include "src/__support/freelist.h"
+#include "src/__support/freetrie.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
namespace LIBC_NAMESPACE_DECL {
-/// A best-fit store of variously-sized free blocks. Blocks can be inserted and
-/// removed in logarithmic time.
-class FreeStore {
- friend class FreeListHeap;
-
-public:
- FreeStore() = default;
- FreeStore(const FreeStore &other) = delete;
- FreeStore &operator=(const FreeStore &other) = delete;
-
- /// Sets the range of possible block sizes. This can only be called when the
- /// trie is empty.
- LIBC_INLINE void set_range(FreeTrie::SizeRange range) {
- large_trie.set_range(range);
- }
-
- /// Insert a free block. If the block is too small to be tracked, nothing
- /// happens.
- void insert(BlockRef block);
+/// Configuration for TLSFFreeStore.
+template <size_t UNIT_SIZE_VAL, size_t STEP_SIZE_BITS_VAL,
+ size_t NUM_STEP_BITS_VAL, size_t NUM_TABLE_ENTRIES_VAL,
+ bool USE_TRIE_FOR_LARGE_FREELIST_VAL = false>
+struct TLSFFreeStoreConfig {
+ static constexpr size_t UNIT_SIZE = UNIT_SIZE_VAL;
+ static constexpr size_t STEP_SIZE_BITS = STEP_SIZE_BITS_VAL;
+ static constexpr size_t NUM_STEP_BITS = NUM_STEP_BITS_VAL;
+ static constexpr size_t NUM_TABLE_ENTRIES = NUM_TABLE_ENTRIES_VAL;
+ static constexpr bool USE_TRIE_FOR_LARGE_FREELIST =
+ USE_TRIE_FOR_LARGE_FREELIST_VAL;
+};
- /// Remove a free block. If the block is too small to be tracked, nothing
- /// happens.
- void remove(BlockRef block);
+// A two-level segregated fit store for free blocks.
+//
+// The store starts with small lists that grow linearly for small sizes, which
+// covers [0, ... UNIT_SIZE * EXP_BASE]. For larger sizes, the bits are managed
+// in a 2-D table. One can think of each row containing NUM_STEPS lists. Along
+// the row, the size grows by 2 exponentially; along the column, the size
+// increases by STEP_SIZE linearly.
+//
+// Mathematical layout:
+// STEP_SIZE = 1 << STEP_SIZE_BITS
+// NUM_STEPS = 1 << NUM_STEP_BITS
+// EXP_BASE = STEP_SIZE * NUM_STEPS
+// LARGE_SIZE_THRESHOLD = UNIT_SIZE * EXP_BASE
+//
+// Visual representation with example parameters:
+// UNIT_SIZE = 32, STEP_SIZE = 8, NUM_STEPS = 4
+// EXP_BASE = 32, THRESHOLD = 1024 B (1 KiB)
+//
+// 1. Small Sizes (Linear Array):
+// Covers [0, ... 1024 B] growing directly by UNIT_SIZE = 32 B
+// +-------+-------+-------+-------+-------+-----------+---------------+
+// | [0 B] | [32B] | [64B] | [96B] | ... | [992 B] | [1024 B (Th)] |
+// +-------+-------+-------+-------+-------+-----------+---------------+
+//
+// 2. Large Sizes (2-D Table):
+// Rows = FL (Exponential growth), Columns = SL (Linear steps)
+// One can think of each Row containing NUM_STEPS (4) lists.
+//
+// LINEAR INCREASE ALONG COLUMN (SL) --->
+// +---------------+---------------+---------------+---------------+
+// | Col = 0 | Col = 1 | Col = 2 | Col = 3 |
+// | (Base) | (+25% FL) | (+50% FL) | (+75% FL) |
+// +---------+---------------+---------------+---------------+---------------+
+// E | Row = 0 | 1024 B | 1280 B | 1536 B | 1792 B |
+// X |(Base 1K)| [1024 - 1279] | [1280 - 1535] | [1536 - 1791] | [1792 - 2047] |
+// P +---------+---------------+---------------+---------------+---------------+
+// O | Row = 1 | 2048 B | 2560 B | 3072 B | 3584 B |
+// N |(Base 2K)| [2048 - 2559] | [2560 - 3071] | [3072 - 3583] | [3584 - 4095] |
+// E +---------+---------------+---------------+---------------+---------------+
+// N | Row = 2 | 4096 B | 5120 B | 6144 B | 7168 B |
+// T |(Base 4K)| [4096 - 5119] | [5120 - 6143] | [6144 - 7167] | [7168 - 8191] |
+// I +---------+---------------+---------------+---------------+---------------+
+// A | Row = 3 | 8192 B | 10240 B | 12288 B | 14336 B |
+// L |(Base 8K)|[8192 - 10239]|[10240 - 12287]|[12288 - 14335]|[14336 - 16383]|
+// +---------+---------------+---------------+---------------+---------------+
+//
+// Note: For the real implementation, we don't actually store the lists in a
+// 2-D structure. Instead, we flatten the entire 2-D layout into a single
+// flat 1-D array of size TOTAL_BITS (free_lists), and map sizes directly to
+// a continuous 1-D index using size_to_bit_index. The allocation state is
+// tracked compactly in the lookup_table bitmask array.
+template <typename CONFIG> class TLSFFreeStoreImpl {
+protected:
+ static_assert(cpp::has_single_bit(CONFIG::UNIT_SIZE),
+ "unit size must be a power of two");
+ static_assert(CONFIG::NUM_TABLE_ENTRIES > 0,
+ "the lookup table must have at least one entry");
- /// Remove a best-fit free block that can contain the given size when
- /// allocated. Returns nullptr if there is no such block.
- BlockRef remove_best_fit(size_t size);
+ static constexpr size_t STEP_SIZE = size_t(1) << CONFIG::STEP_SIZE_BITS;
+ static constexpr size_t NUM_STEPS = size_t(1) << CONFIG::NUM_STEP_BITS;
+ static constexpr size_t EXP_BASE = STEP_SIZE * NUM_STEPS;
+ static constexpr int UNIT_SIZE_LOG2 = cpp::bit_width(CONFIG::UNIT_SIZE) - 1;
+ static constexpr int EXP_BASE_LOG2 =
+ CONFIG::STEP_SIZE_BITS + CONFIG::NUM_STEP_BITS;
+ static constexpr size_t BITS_PER_ENTRY =
+ cpp::numeric_limits<uintptr_t>::digits;
+ static constexpr size_t TOTAL_BITS =
+ CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
+ static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_LARGE_FREELIST;
-private:
+public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
- static constexpr size_t MIN_LARGE_OUTER_SIZE = align_up(
- BlockRef::HEADER_SIZE + sizeof(FreeTrie::Node), BlockRef::MIN_ALIGN);
- static constexpr size_t NUM_SMALL_SIZES =
- (MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / BlockRef::MIN_ALIGN;
+ LIBC_INLINE TLSFFreeStoreImpl() = default;
+ LIBC_INLINE TLSFFreeStoreImpl(const TLSFFreeStoreImpl &other) = delete;
+ LIBC_INLINE TLSFFreeStoreImpl &
+ operator=(const TLSFFreeStoreImpl &other) = delete;
+
+ LIBC_INLINE void insert(BlockRef block);
+ LIBC_INLINE void remove(BlockRef block);
+ LIBC_INLINE BlockRef remove_best_fit(size_t size) {
+ return find_and_remove_fit(size);
+ }
+ LIBC_INLINE BlockRef find_and_remove_fit(size_t size);
+
+protected:
LIBC_INLINE static bool too_small(BlockRef block) {
return block.outer_size() < MIN_OUTER_SIZE;
}
- LIBC_INLINE static bool is_small(BlockRef block) {
- return block.outer_size() < MIN_LARGE_OUTER_SIZE;
- }
- FreeList &small_list(BlockRef block);
- FreeList *find_best_small_fit(size_t size);
+ union ListOrTrie {
+ FreeList list;
+ FreeTrie::Node *trie_root;
+
+ LIBC_INLINE constexpr ListOrTrie() : trie_root(nullptr) {}
+ };
- cpp::array<FreeList, NUM_SMALL_SIZES> small_lists;
- FreeTrie large_trie;
+ cpp::array<uintptr_t, CONFIG::NUM_TABLE_ENTRIES> lookup_table{};
+ cpp::array<ListOrTrie, TOTAL_BITS> free_lists{};
+
+ LIBC_INLINE static constexpr size_t size_to_bit_index(size_t size);
+ LIBC_INLINE void set_bit(size_t bit_index);
+ LIBC_INLINE void clear_bit(size_t bit_index);
+ LIBC_INLINE bool get_bit(size_t bit_index) const;
+ LIBC_INLINE size_t find_first_bit_set_after(size_t bit_index) const;
+ LIBC_INLINE BlockRef remove_first_fit_in_list(size_t index, size_t size);
+ LIBC_INLINE static constexpr FreeTrie::SizeRange index_to_range(size_t index);
+ LIBC_INLINE FreeTrie get_trie(size_t index);
+ LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t index, size_t size);
+ LIBC_INLINE BlockRef pop_min_in_trie(size_t index);
};
-LIBC_INLINE void FreeStore::insert(BlockRef block) {
- if (too_small(block))
- return;
- if (is_small(block))
- small_list(block).push(block);
- else
- large_trie.push(block);
+template <typename CONFIG>
+LIBC_INLINE constexpr size_t
+TLSFFreeStoreImpl<CONFIG>::size_to_bit_index(size_t size) {
+ if (size <= (EXP_BASE << UNIT_SIZE_LOG2))
+ return size >> UNIT_SIZE_LOG2;
+
+ size_t size_ilog2 = static_cast<size_t>(cpp::bit_width(size) - 1);
+ size_t exp_offset = (size_ilog2 - UNIT_SIZE_LOG2 - EXP_BASE_LOG2 - 1)
+ << CONFIG::NUM_STEP_BITS;
+ size_t step_index = size >> (size_ilog2 - CONFIG::NUM_STEP_BITS);
+ size_t index = EXP_BASE + exp_offset + step_index;
+
+ return index < TOTAL_BITS ? index : TOTAL_BITS - 1;
}
-LIBC_INLINE void FreeStore::remove(BlockRef block) {
- if (too_small(block))
- return;
- if (is_small(block)) {
- small_list(block).remove(
- reinterpret_cast<FreeList::Node *>(block.usable_space()));
- } else {
- large_trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::set_bit(size_t bit_index) {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ lookup_table[entry_index] |= uintptr_t(1) << bit_offset;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::clear_bit(size_t bit_index) {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ lookup_table[entry_index] &= ~(uintptr_t(1) << bit_offset);
+}
+
+template <typename CONFIG>
+LIBC_INLINE bool TLSFFreeStoreImpl<CONFIG>::get_bit(size_t bit_index) const {
+ size_t entry_index = bit_index / BITS_PER_ENTRY;
+ size_t bit_offset = bit_index % BITS_PER_ENTRY;
+ return (lookup_table[entry_index] & (uintptr_t(1) << bit_offset)) != 0;
+}
+
+template <typename CONFIG>
+LIBC_INLINE size_t
+TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
+ if (bit_index >= TOTAL_BITS - 1)
+ return TOTAL_BITS;
+
+ size_t target_index = bit_index + 1;
+ size_t start_entry = target_index / BITS_PER_ENTRY;
+ size_t bit_offset = target_index % BITS_PER_ENTRY;
+
+ uintptr_t value = lookup_table[start_entry] & (~uintptr_t(0) << bit_offset);
+ if (value != 0)
+ return start_entry * BITS_PER_ENTRY +
+ static_cast<size_t>(cpp::countr_zero(value));
+
+ for (size_t i = start_entry + 1; i < CONFIG::NUM_TABLE_ENTRIES; ++i) {
+ value = lookup_table[i];
+ if (value != 0)
+ return i * BITS_PER_ENTRY + static_cast<size_t>(cpp::countr_zero(value));
}
+ return TOTAL_BITS;
}
-LIBC_INLINE BlockRef FreeStore::remove_best_fit(size_t size) {
- if (FreeList *list = find_best_small_fit(size)) {
- BlockRef block = list->front();
- list->pop();
- return block;
+template <typename CONFIG>
+LIBC_INLINE constexpr FreeTrie::SizeRange
+TLSFFreeStoreImpl<CONFIG>::index_to_range(size_t index) {
+ LIBC_ASSERT(index >= EXP_BASE && "only call for large lists");
+ size_t local_index = index - EXP_BASE;
+ size_t exp_index = local_index >> CONFIG::NUM_STEP_BITS;
+ size_t linear_index = local_index & (NUM_STEPS - 1);
+
+ size_t row_base = (EXP_BASE << exp_index) << UNIT_SIZE_LOG2;
+ size_t step_size = (STEP_SIZE << exp_index) << UNIT_SIZE_LOG2;
+ size_t min_size = row_base + linear_index * step_size;
+
+ if (index == TOTAL_BITS - 1) {
+ constexpr size_t width = size_t(1)
+ << (cpp::numeric_limits<size_t>::digits - 2);
+ LIBC_ASSERT(min_size < width &&
+ "min_size too large for overflow bin width");
+ return FreeTrie::SizeRange(min_size, width);
}
- if (FreeTrie::Node *best_fit = large_trie.find_best_fit(size)) {
+
+ return FreeTrie::SizeRange(min_size, step_size);
+}
+
+template <typename CONFIG>
+LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie(size_t index) {
+ LIBC_ASSERT(index >= EXP_BASE && "only call for large lists");
+ return FreeTrie(free_lists[index].trie_root, index_to_range(index));
+}
+
+template <typename CONFIG>
+LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit_in_trie(
+ size_t index, size_t size) {
+ FreeTrie trie = get_trie(index);
+ if (FreeTrie::Node *best_fit = trie.find_best_fit(size)) {
BlockRef block = best_fit->block();
- large_trie.remove(best_fit);
+ trie.remove(best_fit);
+ if (trie.empty())
+ clear_bit(index);
return block;
}
return BlockRef();
}
-LIBC_INLINE FreeList &FreeStore::small_list(BlockRef block) {
- LIBC_ASSERT(is_small(block) && "only legal for small blocks");
- return small_lists[(block.outer_size() - MIN_OUTER_SIZE) /
- BlockRef::MIN_ALIGN];
+template <typename CONFIG>
+LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::pop_min_in_trie(size_t index) {
+ FreeTrie trie = get_trie(index);
+ FreeTrie::Node *min_node = trie.pop_min();
+ LIBC_ASSERT(min_node && "bit was set but trie is empty");
+ BlockRef block = min_node->block();
+ if (trie.empty())
+ clear_bit(index);
+ return block;
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
+ if (too_small(block))
+ return;
+ size_t bit_index = size_to_bit_index(block.inner_size());
+
+ if constexpr (USE_TRIE)
+ if (bit_index > EXP_BASE) {
+ get_trie(bit_index).push(block);
+ set_bit(bit_index);
+ return;
+ }
+
+ free_lists[bit_index].list.push(block);
+ set_bit(bit_index);
+}
+
+template <typename CONFIG>
+LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
+ if (too_small(block))
+ return;
+ size_t bit_index = size_to_bit_index(block.inner_size());
+
+ if constexpr (USE_TRIE)
+ if (bit_index > EXP_BASE) {
+ FreeTrie trie = get_trie(bit_index);
+ trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
+ if (trie.empty())
+ clear_bit(bit_index);
+ return;
+ }
+
+ free_lists[bit_index].list.remove(
+ reinterpret_cast<FreeList::Node *>(block.usable_space()));
+ if (free_lists[bit_index].list.empty())
+ clear_bit(bit_index);
+}
+
+template <typename CONFIG>
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
+ FreeList::Node *begin_node = free_lists[index].list.begin();
+ if (begin_node == nullptr)
+ return BlockRef();
+
+ FreeList::Node *cur = begin_node;
+ do {
+ if (cur->size() >= size) {
+ free_lists[index].list.remove(cur);
+ if (free_lists[index].list.empty())
+ clear_bit(index);
+ return cur->block();
+ }
+ cur = cur->next_node();
+ } while (cur != begin_node);
+
+ return BlockRef();
}
-LIBC_INLINE FreeList *FreeStore::find_best_small_fit(size_t size) {
- for (FreeList &list : small_lists)
- if (!list.empty() && list.size() >= size)
- return &list;
- return nullptr;
+template <typename CONFIG>
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
+ size_t bit_index = size_to_bit_index(size);
+
+ if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1)) {
+ if constexpr (USE_TRIE)
+ return find_and_remove_fit_in_trie(TOTAL_BITS - 1, size);
+ else
+ return remove_first_fit_in_list(TOTAL_BITS - 1, size);
+ }
+
+ // 1. Try oversized bins (guaranteed fit, but larger).
+ size_t oversized_bit = find_first_bit_set_after(bit_index);
+ if (LIBC_LIKELY(oversized_bit < TOTAL_BITS)) {
+ if constexpr (USE_TRIE)
+ if (oversized_bit > EXP_BASE)
+ return pop_min_in_trie(oversized_bit);
+
+ BlockRef block = free_lists[oversized_bit].list.front();
+ free_lists[oversized_bit].list.pop();
+ if (free_lists[oversized_bit].list.empty())
+ clear_bit(oversized_bit);
+ return block;
+ }
+
+ // 2. Try exact fit (fallback).
+ if (get_bit(bit_index)) {
+ if constexpr (USE_TRIE) {
+ if (bit_index > EXP_BASE)
+ return find_and_remove_fit_in_trie(bit_index, size);
+ else if (BlockRef block = remove_first_fit_in_list(bit_index, size))
+ return block;
+ } else if (BlockRef block = remove_first_fit_in_list(bit_index, size))
+ return block;
+ }
+
+ return BlockRef();
}
+template <size_t UNIT_SIZE, size_t STEP_SIZE_BITS, size_t NUM_STEP_BITS,
+ size_t NUM_TABLE_ENTRIES, bool USE_TRIE = false>
+using TLSFFreeStore = TLSFFreeStoreImpl<TLSFFreeStoreConfig<
+ UNIT_SIZE, STEP_SIZE_BITS, NUM_STEP_BITS, NUM_TABLE_ENTRIES, USE_TRIE>>;
+
+#ifndef LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST
+#define LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST false
+#endif
+
+using FreeStore =
+ TLSFFreeStore<BlockRef::MIN_ALIGN, 3, 2, (sizeof(uintptr_t) == 8 ? 3 : 6),
+ LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST>;
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 9e35463462b38..9e07f36ad0a53 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -87,15 +87,8 @@ class FreeTrie {
}
};
- LIBC_INLINE constexpr FreeTrie() : FreeTrie(SizeRange{0, 0}) {}
- LIBC_INLINE constexpr FreeTrie(SizeRange range) : range(range) {}
-
- /// Sets the range of possible block sizes. This can only be called when the
- /// trie is empty.
- LIBC_INLINE void set_range(FreeTrie::SizeRange new_range) {
- LIBC_ASSERT(empty() && "cannot change the range of a preexisting trie");
- range = new_range;
- }
+ LIBC_INLINE FreeTrie(Node *&root, SizeRange range)
+ : root(root), range(range) {}
/// @returns Whether the trie contains any blocks.
LIBC_INLINE bool empty() const { return !root; }
@@ -110,15 +103,24 @@ class FreeTrie {
/// nullptr.
Node *find_best_fit(size_t size);
+ /// @returns The node with the minimum size in the trie; otherwise nullptr.
+ LIBC_INLINE Node *find_min();
+
+ /// Removes and returns the node with the minimum size in the trie; otherwise
+ /// nullptr.
+ LIBC_INLINE Node *pop_min();
+
private:
/// @returns Whether a node is the head of its containing freelist.
- bool is_head(Node *node) const { return node->parent || node == root; }
+ LIBC_INLINE bool is_head(Node *node) const {
+ return node->parent || node == root;
+ }
/// Replaces references to one node with another (or nullptr) in all adjacent
/// parent and child nodes.
void replace_node(Node *node, Node *new_node);
- Node *root = nullptr;
+ Node *&root;
SizeRange range;
};
@@ -236,6 +238,47 @@ LIBC_INLINE FreeTrie::Node *FreeTrie::find_best_fit(size_t size) {
}
}
}
+LIBC_INLINE FreeTrie::Node *FreeTrie::find_min() {
+ if (empty())
+ return nullptr;
+
+ Node *cur = root;
+ SizeRange cur_range = range;
+ Node *best_min = nullptr;
+
+ while (cur) {
+ if (cur->lower) {
+ if (cur_range.lower().contains(cur->size())) {
+ if (!best_min || cur->size() < best_min->size()) {
+ best_min = cur;
+ }
+ }
+ cur = cur->lower;
+ cur_range = cur_range.lower();
+ } else {
+ if (cur_range.lower().contains(cur->size())) {
+ if (!best_min || cur->size() < best_min->size()) {
+ best_min = cur;
+ }
+ break;
+ } else {
+ if (!best_min || cur->size() < best_min->size()) {
+ best_min = cur;
+ }
+ cur = cur->upper;
+ cur_range = cur_range.upper();
+ }
+ }
+ }
+ return best_min;
+}
+
+LIBC_INLINE FreeTrie::Node *FreeTrie::pop_min() {
+ Node *min_node = find_min();
+ if (min_node)
+ remove(min_node);
+ return min_node;
+}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index 61103a9126c08..ce655367da00f 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -38,7 +38,6 @@ TEST(LlvmLibcFreeStore, TooSmall) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(too_small);
store.insert(remainder);
@@ -68,21 +67,26 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(smallest);
if (largest_small != smallest)
store.insert(largest_small);
store.insert(remainder);
- // Find exact match for smallest.
- ASSERT_EQ(store.remove_best_fit(smallest.inner_size()).addr(),
- smallest.addr());
- store.insert(smallest);
-
- // Find exact match for largest.
- ASSERT_EQ(store.remove_best_fit(largest_small.inner_size()).addr(),
- largest_small.addr());
- store.insert(largest_small);
+ // For TLSF (oversized first), asking for a size will return the block from
+ // the first non-empty oversized bin if one exists, bypassing the exact bin.
+ if (largest_small != smallest) {
+ BlockRef block = store.remove_best_fit(smallest.inner_size());
+ ASSERT_EQ(block.addr(), largest_small.addr());
+ store.insert(block);
+
+ BlockRef block2 = store.remove_best_fit(largest_small.inner_size());
+ ASSERT_EQ(block2.addr(), remainder.addr());
+ store.insert(block2);
+ } else {
+ BlockRef block = store.remove_best_fit(smallest.inner_size());
+ ASSERT_EQ(block.addr(), remainder.addr());
+ store.insert(block);
+ }
// Search small list for best fit.
BlockRef next_smallest =
@@ -108,7 +112,6 @@ TEST(LlvmLibcFreeStore, Remove) {
BlockRef remainder = *maybeBlock;
FreeStore store;
- store.set_range({0, 4096});
store.insert(small);
store.insert(remainder);
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index bf3284b2faf64..7eea72bcdbc37 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -22,7 +22,9 @@ using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::optional;
TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
- FreeTrie trie({0, 4096});
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
EXPECT_EQ(trie.find_best_fit(123), static_cast<FreeTrie::Node *>(nullptr));
byte mem[1024];
@@ -31,10 +33,10 @@ TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
BlockRef block = *maybeBlock;
trie.push(block);
- FreeTrie::Node *root = trie.find_best_fit(0);
- ASSERT_EQ(root->block().addr(), block.addr());
- EXPECT_EQ(trie.find_best_fit(block.inner_size() - 1), root);
- EXPECT_EQ(trie.find_best_fit(block.inner_size()), root);
+ FreeTrie::Node *found = trie.find_best_fit(0);
+ ASSERT_EQ(found->block().addr(), block.addr());
+ EXPECT_EQ(trie.find_best_fit(block.inner_size() - 1), found);
+ EXPECT_EQ(trie.find_best_fit(block.inner_size()), found);
EXPECT_EQ(trie.find_best_fit(block.inner_size() + 1),
static_cast<FreeTrie::Node *>(nullptr));
EXPECT_EQ(trie.find_best_fit(4095), static_cast<FreeTrie::Node *>(nullptr));
@@ -47,10 +49,12 @@ TEST(LlvmLibcFreeTrie, FindBestFitLower) {
BlockRef lower = *maybeBlock;
maybeBlock = lower.split(512);
ASSERT_TRUE(maybeBlock.has_value());
- BlockRef root = *maybeBlock;
+ BlockRef root_block = *maybeBlock;
- FreeTrie trie({0, 4096});
- trie.push(root);
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
+ trie.push(root_block);
trie.push(lower);
EXPECT_EQ(trie.find_best_fit(0)->block().addr(), lower.addr());
@@ -60,36 +64,40 @@ TEST(LlvmLibcFreeTrie, FindBestFitUpper) {
byte mem[4096];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
- BlockRef root = *maybeBlock;
- maybeBlock = root.split(512);
+ BlockRef root_block = *maybeBlock;
+ maybeBlock = root_block.split(512);
ASSERT_TRUE(maybeBlock.has_value());
BlockRef upper = *maybeBlock;
- FreeTrie trie({0, 4096});
- trie.push(root);
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
+ trie.push(root_block);
trie.push(upper);
- EXPECT_EQ(trie.find_best_fit(root.inner_size() + 1)->block().addr(),
+ EXPECT_EQ(trie.find_best_fit(root_block.inner_size() + 1)->block().addr(),
upper.addr());
// The upper subtrie should be skipped if it could not contain a better fit.
- EXPECT_EQ(trie.find_best_fit(root.inner_size() - 1)->block().addr(),
- root.addr());
+ EXPECT_EQ(trie.find_best_fit(root_block.inner_size() - 1)->block().addr(),
+ root_block.addr());
}
TEST(LlvmLibcFreeTrie, FindBestFitLowerAndUpper) {
byte mem[4096];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
- BlockRef root = *maybeBlock;
- maybeBlock = root.split(1024);
+ BlockRef root_block = *maybeBlock;
+ maybeBlock = root_block.split(1024);
ASSERT_TRUE(maybeBlock.has_value());
BlockRef lower = *maybeBlock;
maybeBlock = lower.split(128);
ASSERT_TRUE(maybeBlock.has_value());
BlockRef upper = *maybeBlock;
- FreeTrie trie({0, 4096});
- trie.push(root);
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
+ trie.push(root_block);
trie.push(lower);
trie.push(upper);
@@ -114,7 +122,9 @@ TEST(LlvmLibcFreeTrie, Remove) {
BlockRef large = *maybeBlock;
// Removing the root empties the trie.
- FreeTrie trie({0, 4096});
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
trie.push(large);
FreeTrie::Node *large_node = trie.find_best_fit(0);
ASSERT_EQ(large_node->block().addr(), large.addr());
@@ -132,3 +142,46 @@ TEST(LlvmLibcFreeTrie, Remove) {
EXPECT_EQ(trie.find_best_fit(large.inner_size())->block().addr(),
large.addr());
}
+
+TEST(LlvmLibcFreeTrie, PopMin) {
+ alignas(BlockRef::MIN_ALIGN) byte mem[4096];
+ optional<BlockRef> maybe_block = BlockRef::init(mem);
+ ASSERT_TRUE(maybe_block.has_value());
+ BlockRef root_block = *maybe_block;
+ maybe_block = root_block.split(1024);
+ ASSERT_TRUE(maybe_block.has_value());
+ BlockRef lower = *maybe_block;
+ maybe_block = lower.split(128);
+ ASSERT_TRUE(maybe_block.has_value());
+ BlockRef upper = *maybe_block;
+
+ FreeTrie::Node *root = nullptr;
+ FreeTrie::SizeRange range{0, 4096};
+ FreeTrie trie(root, range);
+
+ // Empty pop
+ EXPECT_EQ(trie.pop_min(), static_cast<FreeTrie::Node *>(nullptr));
+
+ trie.push(root_block);
+ trie.push(lower);
+ trie.push(upper);
+
+ // Min should be lower (~128)
+ FreeTrie::Node *min1 = trie.pop_min();
+ ASSERT_NE(min1, static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_EQ(min1->block().addr(), lower.addr());
+
+ // Next min should be root_block (~1024)
+ FreeTrie::Node *min2 = trie.pop_min();
+ ASSERT_NE(min2, static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_EQ(min2->block().addr(), root_block.addr());
+
+ // Next min should be upper (~2944)
+ FreeTrie::Node *min3 = trie.pop_min();
+ ASSERT_NE(min3, static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_EQ(min3->block().addr(), upper.addr());
+
+ // Now empty
+ EXPECT_EQ(trie.pop_min(), static_cast<FreeTrie::Node *>(nullptr));
+ EXPECT_TRUE(trie.empty());
+}
>From a193a91eb59cad755f51d35e4672bea893f136c0 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Thu, 11 Jun 2026 15:23:40 -0700
Subject: [PATCH 2/4] [libc] restrict FreeTrie to the overflow bin in TLSF
- Rename config option to USE_TRIE_FOR_OVERFLOW_BIN.
- Restrict FreeTrie usage to only the last overflow bin (bit_index == TOTAL_BITS - 1).
- Simplify exact fit search to only use FreeList, as large sizes now always go to the overflow bin.
- Simplify away index_to_range by using a fixed [0, INF) range for the overflow Trie.
- Remove index argument from Trie helper functions as they only operate on the overflow bin.
TAG=agy
CONV=fe3b4efa-7a5b-4c74-8257-e53f0d6e4850
---
libc/src/__support/freestore.h | 90 ++++++++++++----------------------
1 file changed, 32 insertions(+), 58 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 4fb90cabf47bd..13b79c0debf39 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -30,14 +30,14 @@ namespace LIBC_NAMESPACE_DECL {
/// Configuration for TLSFFreeStore.
template <size_t UNIT_SIZE_VAL, size_t STEP_SIZE_BITS_VAL,
size_t NUM_STEP_BITS_VAL, size_t NUM_TABLE_ENTRIES_VAL,
- bool USE_TRIE_FOR_LARGE_FREELIST_VAL = false>
+ bool USE_TRIE_FOR_OVERFLOW_BIN_VAL = false>
struct TLSFFreeStoreConfig {
static constexpr size_t UNIT_SIZE = UNIT_SIZE_VAL;
static constexpr size_t STEP_SIZE_BITS = STEP_SIZE_BITS_VAL;
static constexpr size_t NUM_STEP_BITS = NUM_STEP_BITS_VAL;
static constexpr size_t NUM_TABLE_ENTRIES = NUM_TABLE_ENTRIES_VAL;
- static constexpr bool USE_TRIE_FOR_LARGE_FREELIST =
- USE_TRIE_FOR_LARGE_FREELIST_VAL;
+ static constexpr bool USE_TRIE_FOR_OVERFLOW_BIN =
+ USE_TRIE_FOR_OVERFLOW_BIN_VAL;
};
// A two-level segregated fit store for free blocks.
@@ -108,7 +108,9 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
cpp::numeric_limits<uintptr_t>::digits;
static constexpr size_t TOTAL_BITS =
CONFIG::NUM_TABLE_ENTRIES * BITS_PER_ENTRY;
- static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_LARGE_FREELIST;
+ static constexpr bool USE_TRIE = CONFIG::USE_TRIE_FOR_OVERFLOW_BIN;
+ static constexpr size_t OVERFLOW_WIDTH =
+ size_t(1) << (cpp::numeric_limits<size_t>::digits - 2);
public:
static constexpr size_t MIN_OUTER_SIZE = align_up(
@@ -147,10 +149,9 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
LIBC_INLINE bool get_bit(size_t bit_index) const;
LIBC_INLINE size_t find_first_bit_set_after(size_t bit_index) const;
LIBC_INLINE BlockRef remove_first_fit_in_list(size_t index, size_t size);
- LIBC_INLINE static constexpr FreeTrie::SizeRange index_to_range(size_t index);
- LIBC_INLINE FreeTrie get_trie(size_t index);
- LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t index, size_t size);
- LIBC_INLINE BlockRef pop_min_in_trie(size_t index);
+ LIBC_INLINE FreeTrie get_trie();
+ LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t size);
+ LIBC_INLINE BlockRef pop_min_in_trie();
};
template <typename CONFIG>
@@ -213,56 +214,33 @@ TLSFFreeStoreImpl<CONFIG>::find_first_bit_set_after(size_t bit_index) const {
}
template <typename CONFIG>
-LIBC_INLINE constexpr FreeTrie::SizeRange
-TLSFFreeStoreImpl<CONFIG>::index_to_range(size_t index) {
- LIBC_ASSERT(index >= EXP_BASE && "only call for large lists");
- size_t local_index = index - EXP_BASE;
- size_t exp_index = local_index >> CONFIG::NUM_STEP_BITS;
- size_t linear_index = local_index & (NUM_STEPS - 1);
-
- size_t row_base = (EXP_BASE << exp_index) << UNIT_SIZE_LOG2;
- size_t step_size = (STEP_SIZE << exp_index) << UNIT_SIZE_LOG2;
- size_t min_size = row_base + linear_index * step_size;
-
- if (index == TOTAL_BITS - 1) {
- constexpr size_t width = size_t(1)
- << (cpp::numeric_limits<size_t>::digits - 2);
- LIBC_ASSERT(min_size < width &&
- "min_size too large for overflow bin width");
- return FreeTrie::SizeRange(min_size, width);
- }
-
- return FreeTrie::SizeRange(min_size, step_size);
-}
-
-template <typename CONFIG>
-LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie(size_t index) {
- LIBC_ASSERT(index >= EXP_BASE && "only call for large lists");
- return FreeTrie(free_lists[index].trie_root, index_to_range(index));
+LIBC_INLINE FreeTrie TLSFFreeStoreImpl<CONFIG>::get_trie() {
+ return FreeTrie(free_lists[TOTAL_BITS - 1].trie_root,
+ FreeTrie::SizeRange(0, OVERFLOW_WIDTH));
}
template <typename CONFIG>
-LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit_in_trie(
- size_t index, size_t size) {
- FreeTrie trie = get_trie(index);
+LIBC_INLINE BlockRef
+TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit_in_trie(size_t size) {
+ FreeTrie trie = get_trie();
if (FreeTrie::Node *best_fit = trie.find_best_fit(size)) {
BlockRef block = best_fit->block();
trie.remove(best_fit);
if (trie.empty())
- clear_bit(index);
+ clear_bit(TOTAL_BITS - 1);
return block;
}
return BlockRef();
}
template <typename CONFIG>
-LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::pop_min_in_trie(size_t index) {
- FreeTrie trie = get_trie(index);
+LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::pop_min_in_trie() {
+ FreeTrie trie = get_trie();
FreeTrie::Node *min_node = trie.pop_min();
LIBC_ASSERT(min_node && "bit was set but trie is empty");
BlockRef block = min_node->block();
if (trie.empty())
- clear_bit(index);
+ clear_bit(TOTAL_BITS - 1);
return block;
}
@@ -273,8 +251,8 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
size_t bit_index = size_to_bit_index(block.inner_size());
if constexpr (USE_TRIE)
- if (bit_index > EXP_BASE) {
- get_trie(bit_index).push(block);
+ if (bit_index == TOTAL_BITS - 1) {
+ get_trie().push(block);
set_bit(bit_index);
return;
}
@@ -290,8 +268,8 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
size_t bit_index = size_to_bit_index(block.inner_size());
if constexpr (USE_TRIE)
- if (bit_index > EXP_BASE) {
- FreeTrie trie = get_trie(bit_index);
+ if (bit_index == TOTAL_BITS - 1) {
+ FreeTrie trie = get_trie();
trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
if (trie.empty())
clear_bit(bit_index);
@@ -332,7 +310,7 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1)) {
if constexpr (USE_TRIE)
- return find_and_remove_fit_in_trie(TOTAL_BITS - 1, size);
+ return find_and_remove_fit_in_trie(size);
else
return remove_first_fit_in_list(TOTAL_BITS - 1, size);
}
@@ -340,9 +318,10 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
// 1. Try oversized bins (guaranteed fit, but larger).
size_t oversized_bit = find_first_bit_set_after(bit_index);
if (LIBC_LIKELY(oversized_bit < TOTAL_BITS)) {
- if constexpr (USE_TRIE)
- if (oversized_bit > EXP_BASE)
- return pop_min_in_trie(oversized_bit);
+ if constexpr (USE_TRIE) {
+ if (oversized_bit == TOTAL_BITS - 1)
+ return pop_min_in_trie();
+ }
BlockRef block = free_lists[oversized_bit].list.front();
free_lists[oversized_bit].list.pop();
@@ -353,12 +332,7 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
// 2. Try exact fit (fallback).
if (get_bit(bit_index)) {
- if constexpr (USE_TRIE) {
- if (bit_index > EXP_BASE)
- return find_and_remove_fit_in_trie(bit_index, size);
- else if (BlockRef block = remove_first_fit_in_list(bit_index, size))
- return block;
- } else if (BlockRef block = remove_first_fit_in_list(bit_index, size))
+ if (BlockRef block = remove_first_fit_in_list(bit_index, size))
return block;
}
@@ -370,13 +344,13 @@ template <size_t UNIT_SIZE, size_t STEP_SIZE_BITS, size_t NUM_STEP_BITS,
using TLSFFreeStore = TLSFFreeStoreImpl<TLSFFreeStoreConfig<
UNIT_SIZE, STEP_SIZE_BITS, NUM_STEP_BITS, NUM_TABLE_ENTRIES, USE_TRIE>>;
-#ifndef LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST
-#define LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST false
+#ifndef LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN
+#define LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN false
#endif
using FreeStore =
TLSFFreeStore<BlockRef::MIN_ALIGN, 3, 2, (sizeof(uintptr_t) == 8 ? 3 : 6),
- LIBC_COPT_USE_TRIE_FOR_LARGE_FREELIST>;
+ LIBC_COPT_USE_TRIE_FOR_OVERFLOW_BIN>;
} // namespace LIBC_NAMESPACE_DECL
>From 9dd6ebe8adcf97d068a18c59e71e5af9d455cd03 Mon Sep 17 00:00:00 2001
From: yfzhu <yfzhu at google.com>
Date: Tue, 23 Jun 2026 09:15:24 -0700
Subject: [PATCH 3/4] [libc] make baremetal freelist heap headers-only
This change moves the implementations of FreeList, FreeTrie and FreeListHeap
to their respective header files, making the baremetal freelist heap library
headers-only. This simplifies integration and avoids compilation issues.
TAG=agy
CONV=4118a58d-30c6-4fd4-8e45-defe707d8bba
---
libc/src/__support/CMakeLists.txt | 15 +--
libc/src/__support/freelist.cpp | 47 ----------
libc/src/__support/freelist.h | 31 ++++++-
libc/src/__support/freelist_heap.cpp | 19 ----
libc/src/__support/freelist_heap.h | 26 ++++--
libc/src/__support/freetrie.cpp | 64 -------------
libc/src/__support/freetrie.h | 56 ++++++++++-
.../llvm-project-overlay/libc/BUILD.bazel | 92 +++++++++++++++++++
8 files changed, 194 insertions(+), 156 deletions(-)
delete mode 100644 libc/src/__support/freelist.cpp
delete mode 100644 libc/src/__support/freelist_heap.cpp
delete mode 100644 libc/src/__support/freetrie.cpp
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 9d08b4bcf7303..2c5f38a8d2f72 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -25,12 +25,10 @@ add_header_library(
libc.src.__support.math_extras
)
-add_object_library(
+add_header_library(
freelist
HDRS
freelist.h
- SRCS
- freelist.cpp
DEPENDS
.block
libc.src.__support.fixedvector
@@ -40,12 +38,10 @@ add_object_library(
libc.src.__support.CPP.span
)
-add_object_library(
+add_header_library(
freetrie
HDRS
freetrie.h
- SRCS
- freetrie.cpp
DEPENDS
.block
.freelist
@@ -59,15 +55,10 @@ add_header_library(
.freetrie
)
-libc_set_definition(libc_freelist_malloc_size "LIBC_FREELIST_MALLOC_SIZE=${LIBC_CONF_FREELIST_MALLOC_BUFFER_SIZE}")
-add_object_library(
+add_header_library(
freelist_heap
- SRCS
- freelist_heap.cpp
HDRS
freelist_heap.h
- COMPILE_OPTIONS
- ${libc_freelist_malloc_size}
DEPENDS
.block
.freelist
diff --git a/libc/src/__support/freelist.cpp b/libc/src/__support/freelist.cpp
deleted file mode 100644
index 28f73b2726d7d..0000000000000
--- a/libc/src/__support/freelist.cpp
+++ /dev/null
@@ -1,47 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// Implementation for freelist.
-///
-//===----------------------------------------------------------------------===//
-
-#include "freelist.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-void FreeList::push(Node *node) {
- if (begin_) {
- LIBC_ASSERT(BlockRef::from_usable_space(node).outer_size() ==
- begin_->block().outer_size() &&
- "freelist entries must have the same size");
- // Since the list is circular, insert the node immediately before begin_.
- node->prev = begin_->prev;
- node->next = begin_;
- begin_->prev->next = node;
- begin_->prev = node;
- } else {
- begin_ = node->prev = node->next = node;
- }
-}
-
-void FreeList::remove(Node *node) {
- LIBC_ASSERT(begin_ && "cannot remove from empty list");
- if (node == node->next) {
- LIBC_ASSERT(node == begin_ &&
- "a self-referential node must be the only element");
- begin_ = nullptr;
- } else {
- node->prev->next = node->next;
- node->next->prev = node->prev;
- if (begin_ == node)
- begin_ = node->next;
- }
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index ae2de684b3a24..34d8f759fd0bb 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -15,6 +15,8 @@
#define LLVM_LIBC_SRC___SUPPORT_FREELIST_H
#include "block.h"
+#include "src/__support/libc_assert.h"
+#include "src/__support/macros/config.h"
namespace LIBC_NAMESPACE_DECL {
@@ -80,13 +82,38 @@ class FreeList {
/// Push an already-constructed node to the back of the list.
/// This allows pushing derived node types with additional data.
- void push(Node *node);
+ LIBC_INLINE void push(Node *node) {
+ if (begin_) {
+ LIBC_ASSERT(BlockRef::from_usable_space(node).outer_size() ==
+ begin_->block().outer_size() &&
+ "freelist entries must have the same size");
+ // Since the list is circular, insert the node immediately before begin_.
+ node->prev = begin_->prev;
+ node->next = begin_;
+ begin_->prev->next = node;
+ begin_->prev = node;
+ } else {
+ begin_ = node->prev = node->next = node;
+ }
+ }
/// Pop the first node from the list.
LIBC_INLINE void pop() { remove(begin_); }
/// Remove an arbitrary node from the list.
- void remove(Node *node);
+ LIBC_INLINE void remove(Node *node) {
+ LIBC_ASSERT(begin_ && "cannot remove from empty list");
+ if (node == node->next) {
+ LIBC_ASSERT(node == begin_ &&
+ "a self-referential node must be the only element");
+ begin_ = nullptr;
+ } else {
+ node->prev->next = node->next;
+ node->next->prev = node->prev;
+ if (begin_ == node)
+ begin_ = node->next;
+ }
+ }
private:
Node *begin_;
diff --git a/libc/src/__support/freelist_heap.cpp b/libc/src/__support/freelist_heap.cpp
deleted file mode 100644
index 4deb0e0f09e22..0000000000000
--- a/libc/src/__support/freelist_heap.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation for freelist_heap ----------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/__support/freelist_heap.h"
-#include "src/__support/macros/config.h"
-
-#include <stddef.h>
-
-namespace LIBC_NAMESPACE_DECL {
-
-static LIBC_CONSTINIT FreeListHeap freelist_heap_symbols;
-FreeListHeap *freelist_heap = &freelist_heap_symbols;
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 976a6eec4023e..c05bf806990c8 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -21,6 +21,7 @@
#include "src/__support/CPP/optional.h"
#include "src/__support/CPP/span.h"
#include "src/__support/libc_assert.h"
+#include "src/__support/macros/attributes.h"
#include "src/__support/macros/config.h"
#include "src/__support/math_extras.h"
#include "src/string/memory_utils/inline_memcpy.h"
@@ -80,7 +81,7 @@ template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
cpp::byte buffer[BUFF_SIZE];
};
-LIBC_INLINE void FreeListHeap::init() {
+[[gnu::noinline]] LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
@@ -88,7 +89,8 @@ LIBC_INLINE void FreeListHeap::init() {
is_initialized = true;
}
-LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *
+FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (size == 0)
return nullptr;
@@ -113,12 +115,12 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
return block_info.block.usable_space();
}
-LIBC_INLINE void *FreeListHeap::allocate(size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *FreeListHeap::allocate(size_t size) {
return allocate_impl(BlockRef::MIN_ALIGN, size);
}
-LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment,
- size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *
+FreeListHeap::aligned_allocate(size_t alignment, size_t size) {
// The alignment must be an integral power of two.
if (!IsPow2(alignment))
return nullptr;
@@ -133,7 +135,7 @@ LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment,
return allocate_impl(alignment, size);
}
-LIBC_INLINE void FreeListHeap::free(void *ptr) {
+[[gnu::noinline]] LIBC_INLINE void FreeListHeap::free(void *ptr) {
if (ptr == nullptr)
return;
@@ -164,7 +166,8 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
free_store.insert(block);
}
-LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
+[[gnu::noinline]] LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block,
+ size_t size) {
size_t min_outer_size = BlockRef::outer_size(cpp::max(size, sizeof(size_t)));
uintptr_t next_block_start = BlockRef::next_possible_block_start(
block.addr() + min_outer_size, BlockRef::MIN_ALIGN);
@@ -193,7 +196,8 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
// Follows constract of the C standard realloc() function
// If ptr is free'd, will return nullptr.
-LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *FreeListHeap::realloc(void *ptr,
+ size_t size) {
if (size == 0) {
free(ptr);
return nullptr;
@@ -228,7 +232,8 @@ LIBC_INLINE void *FreeListHeap::realloc(void *ptr, size_t size) {
return new_ptr;
}
-LIBC_INLINE void *FreeListHeap::calloc(size_t num, size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *FreeListHeap::calloc(size_t num,
+ size_t size) {
size_t bytes;
if (__builtin_mul_overflow(num, size, &bytes))
return nullptr;
@@ -238,7 +243,8 @@ LIBC_INLINE void *FreeListHeap::calloc(size_t num, size_t size) {
return ptr;
}
-extern FreeListHeap *freelist_heap;
+LIBC_INLINE_VAR LIBC_CONSTINIT FreeListHeap freelist_heap_symbols;
+LIBC_INLINE_VAR FreeListHeap *freelist_heap = &freelist_heap_symbols;
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freetrie.cpp b/libc/src/__support/freetrie.cpp
deleted file mode 100644
index e76efe717f215..0000000000000
--- a/libc/src/__support/freetrie.cpp
+++ /dev/null
@@ -1,64 +0,0 @@
-//===-- Implementation for freetrie ---------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "freetrie.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-void FreeTrie::remove(Node *node) {
- LIBC_ASSERT(!empty() && "cannot remove from empty trie");
- FreeList list = node;
- list.pop();
- Node *new_node = static_cast<Node *>(list.begin());
- if (!new_node) {
- // The freelist is empty. Replace the subtrie root with an arbitrary leaf.
- // This is legal because there is no relationship between the size of the
- // root and its children.
- Node *leaf = node;
- while (leaf->lower || leaf->upper)
- leaf = leaf->lower ? leaf->lower : leaf->upper;
- if (leaf == node) {
- // If the root is a leaf, then removing it empties the subtrie.
- replace_node(node, nullptr);
- return;
- }
-
- replace_node(leaf, nullptr);
- new_node = leaf;
- }
-
- if (!is_head(node))
- return;
-
- // Copy the trie links to the new head.
- new_node->lower = node->lower;
- new_node->upper = node->upper;
- new_node->parent = node->parent;
- replace_node(node, new_node);
-}
-
-void FreeTrie::replace_node(Node *node, Node *new_node) {
- LIBC_ASSERT(is_head(node) && "only head nodes contain trie links");
-
- if (node->parent) {
- Node *&parent_child =
- node->parent->lower == node ? node->parent->lower : node->parent->upper;
- LIBC_ASSERT(parent_child == node &&
- "no reference to child node found in parent");
- parent_child = new_node;
- } else {
- LIBC_ASSERT(root == node && "non-root node had no parent");
- root = new_node;
- }
- if (node->lower)
- node->lower->parent = new_node;
- if (node->upper)
- node->upper->parent = new_node;
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 9e07f36ad0a53..6c1b479ba8813 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -15,6 +15,7 @@
#define LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
#include "freelist.h"
+#include "src/__support/libc_assert.h"
namespace LIBC_NAMESPACE_DECL {
@@ -97,7 +98,7 @@ class FreeTrie {
void push(BlockRef block);
/// Remove a node from this trie node's free list.
- void remove(Node *node);
+ LIBC_INLINE void remove(Node *node);
/// @returns A smallest node that can allocate the given size; otherwise
/// nullptr.
@@ -118,7 +119,7 @@ class FreeTrie {
/// Replaces references to one node with another (or nullptr) in all adjacent
/// parent and child nodes.
- void replace_node(Node *node, Node *new_node);
+ LIBC_INLINE void replace_node(Node *node, Node *new_node);
Node *&root;
SizeRange range;
@@ -280,6 +281,57 @@ LIBC_INLINE FreeTrie::Node *FreeTrie::pop_min() {
return min_node;
}
+LIBC_INLINE void FreeTrie::remove(Node *node) {
+ LIBC_ASSERT(!empty() && "cannot remove from empty trie");
+ FreeList list = node;
+ list.pop();
+ Node *new_node = static_cast<Node *>(list.begin());
+ if (!new_node) {
+ // The freelist is empty. Replace the subtrie root with an arbitrary leaf.
+ // This is legal because there is no relationship between the size of the
+ // root and its children.
+ Node *leaf = node;
+ while (leaf->lower || leaf->upper)
+ leaf = leaf->lower ? leaf->lower : leaf->upper;
+ if (leaf == node) {
+ // If the root is a leaf, then removing it empties the subtrie.
+ replace_node(node, nullptr);
+ return;
+ }
+
+ replace_node(leaf, nullptr);
+ new_node = leaf;
+ }
+
+ if (!is_head(node))
+ return;
+
+ // Copy the trie links to the new head.
+ new_node->lower = node->lower;
+ new_node->upper = node->upper;
+ new_node->parent = node->parent;
+ replace_node(node, new_node);
+}
+
+LIBC_INLINE void FreeTrie::replace_node(Node *node, Node *new_node) {
+ LIBC_ASSERT(is_head(node) && "only head nodes contain trie links");
+
+ if (node->parent) {
+ Node *&parent_child =
+ node->parent->lower == node ? node->parent->lower : node->parent->upper;
+ LIBC_ASSERT(parent_child == node &&
+ "no reference to child node found in parent");
+ parent_child = new_node;
+ } else {
+ LIBC_ASSERT(root == node && "non-root node had no parent");
+ root = new_node;
+ }
+ if (node->lower)
+ node->lower->parent = new_node;
+ if (node->upper)
+ node->upper->parent = new_node;
+}
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 7c0a9faff678d..623d9f0858eac 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -17,6 +17,7 @@ load(
"libc_math_function",
"libc_support_library",
)
+load(":libc_namespace.bzl", "LIBC_NAMESPACE")
load(":platforms.bzl", "PLATFORM_CPU_ARM64", "PLATFORM_CPU_X86_64")
package(
@@ -16250,3 +16251,94 @@ libc_function(
":types_wchar_t",
],
)
+
+libc_support_library(
+ name = "__support_block",
+ hdrs = ["src/__support/block.h"],
+ deps = [
+ ":__support_common",
+ ":__support_cpp_algorithm",
+ ":__support_cpp_cstddef",
+ ":__support_cpp_limits",
+ ":__support_cpp_new",
+ ":__support_cpp_optional",
+ ":__support_cpp_span",
+ ":__support_libc_assert",
+ ":__support_macros_attributes",
+ ":__support_macros_config",
+ ":__support_math_extras",
+ ":hdr_stdint_proxy",
+ ":string_memory_utils",
+ ":types_size_t",
+ ],
+)
+
+libc_support_library(
+ name = "freelist",
+ hdrs = ["src/__support/freelist.h"],
+ deps = [
+ ":__support_block",
+ ":__support_common",
+ ":__support_libc_assert",
+ ":__support_macros_config",
+ ],
+)
+
+libc_support_library(
+ name = "freetrie",
+ hdrs = ["src/__support/freetrie.h"],
+ deps = [
+ ":__support_common",
+ ":__support_libc_assert",
+ ":__support_macros_config",
+ ":freelist",
+ ],
+)
+
+libc_support_library(
+ name = "freestore",
+ hdrs = ["src/__support/freestore.h"],
+ deps = [
+ ":__support_block",
+ ":__support_common",
+ ":__support_cpp_array",
+ ":__support_cpp_bit",
+ ":__support_cpp_limits",
+ ":__support_macros_config",
+ ":__support_macros_optimization",
+ ":freelist",
+ ":freetrie",
+ ":hdr_stdint_proxy",
+ ":types_size_t",
+ ],
+)
+
+libc_support_library(
+ name = "freelist_heap",
+ hdrs = ["src/__support/freelist_heap.h"],
+ deps = [
+ ":__support_block",
+ ":__support_common",
+ ":__support_cpp_optional",
+ ":__support_cpp_span",
+ ":__support_libc_assert",
+ ":__support_macros_attributes",
+ ":__support_macros_config",
+ ":__support_math_extras",
+ ":freestore",
+ ":string_memory_utils",
+ ],
+)
+
+cc_library(
+ name = "freelist_heap_lib",
+ defines = [
+ "LLVM_LIBC_SRC___SUPPORT_CPP_NEW_H",
+ "LIBC_NAMESPACE=" + LIBC_NAMESPACE,
+ ],
+ includes = ["."],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":freelist_heap",
+ ],
+)
>From af4d9b03639edfcc4b12d7adc4a7dd322baeaae0 Mon Sep 17 00:00:00 2001
From: yfzhu <yfzhu at google.com>
Date: Tue, 23 Jun 2026 09:17:39 -0700
Subject: [PATCH 4/4] [libc] add option for hardened freelist
This change adds the LIBC_COPT_HARDEN_FREELIST option which enables
encryption of forward (next) and backward (prev) pointers in the baremetal
freelist heap. When enabled, the FreeListHeap class stores three const
uintptr_t keys, which are initialized in its constructors. These keys are
passed down to FreeStore and FreeList operations to perform encoding and
corruption verification. If corruption is detected, the allocator will trap.
TAG=agy
CONV=4118a58d-30c6-4fd4-8e45-defe707d8bba
---
libc/src/__support/freelist.h | 130 ++++++++++++++----
libc/src/__support/freelist_heap.h | 59 ++++++--
libc/src/__support/freestore.h | 45 +++---
libc/src/__support/freetrie.h | 4 +-
.../test/src/__support/freelist_heap_test.cpp | 21 ++-
libc/test/src/__support/freelist_test.cpp | 83 +++++++++--
.../llvm-project-overlay/libc/BUILD.bazel | 1 +
.../libc/test/src/__support/BUILD.bazel | 50 +++++++
8 files changed, 326 insertions(+), 67 deletions(-)
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index 34d8f759fd0bb..9ba3bac041894 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -15,11 +15,73 @@
#define LLVM_LIBC_SRC___SUPPORT_FREELIST_H
#include "block.h"
+#include "hdr/stdint_proxy.h"
#include "src/__support/libc_assert.h"
#include "src/__support/macros/config.h"
+#ifndef LIBC_COPT_HARDEN_FREELIST
+#define LIBC_COPT_HARDEN_FREELIST false
+#endif
+
+#if LIBC_COPT_HARDEN_FREELIST
+#define LIBC_HARDENING_ASSERT(cond) \
+ do { \
+ if (LIBC_UNLIKELY(!(cond))) { \
+ __builtin_trap(); \
+ } \
+ } while (0)
+#else
+#define LIBC_HARDENING_ASSERT(cond) LIBC_ASSERT(cond)
+#endif
+
namespace LIBC_NAMESPACE_DECL {
+struct FreeListSecrets {
+#if LIBC_COPT_HARDEN_FREELIST
+ uintptr_t k0;
+ uintptr_t k1;
+ uintptr_t k2;
+#endif
+
+ template <typename T> LIBC_INLINE T *decrypt_next(T *next_val) const {
+#if LIBC_COPT_HARDEN_FREELIST
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(next_val) ^ k0);
+#else
+ return next_val;
+#endif
+ }
+
+ template <typename T>
+ LIBC_INLINE T *decrypt_prev([[maybe_unused]] const void *node,
+ T *prev_val) const {
+#if LIBC_COPT_HARDEN_FREELIST
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(prev_val) ^ k1 ^
+ reinterpret_cast<uintptr_t>(node) ^ k2);
+#else
+ return prev_val;
+#endif
+ }
+
+ template <typename T> LIBC_INLINE T *encrypt_next(T *next_val) const {
+#if LIBC_COPT_HARDEN_FREELIST
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(next_val) ^ k0);
+#else
+ return next_val;
+#endif
+ }
+
+ template <typename T>
+ LIBC_INLINE T *encrypt_prev([[maybe_unused]] const void *node,
+ T *prev_val) const {
+#if LIBC_COPT_HARDEN_FREELIST
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(prev_val) ^ k1 ^
+ reinterpret_cast<uintptr_t>(node) ^ k2);
+#else
+ return prev_val;
+#endif
+ }
+};
+
/// A circularly-linked FIFO list storing free Blocks. All Blocks on a list
/// are the same size. The blocks are referenced by Nodes in the list; the list
/// refers to these, but it does not own them.
@@ -43,9 +105,6 @@ class FreeList {
/// @returns The inner size of blocks in the list containing this node.
LIBC_INLINE size_t size() const { return block().inner_size(); }
- /// @returns The next node in the list containing this node.
- LIBC_INLINE Node *next_node() const { return next; }
-
private:
// Circularly linked pointers to adjacent nodes.
Node *prev;
@@ -53,7 +112,7 @@ class FreeList {
friend class FreeList;
};
- LIBC_INLINE constexpr FreeList() : FreeList(nullptr) {}
+ LIBC_INLINE constexpr FreeList() : begin_(nullptr) {}
LIBC_INLINE constexpr FreeList(Node *begin) : begin_(begin) {}
LIBC_INLINE bool empty() const { return !begin_; }
@@ -70,48 +129,73 @@ class FreeList {
/// @returns The first block in the list.
LIBC_INLINE BlockRef front() { return begin_->block(); }
+ LIBC_INLINE Node *next_node(const Node *node,
+ const FreeListSecrets &secrets) const {
+ return node ? secrets.decrypt_next(node->next) : nullptr;
+ }
+
+ LIBC_INLINE Node *prev_node(const Node *node,
+ const FreeListSecrets &secrets) const {
+ return node ? secrets.decrypt_prev(node, node->prev) : nullptr;
+ }
+
/// Push a block to the back of the list.
/// The block must be large enough to contain a node.
- LIBC_INLINE void push(BlockRef block) {
+ LIBC_INLINE void push(BlockRef block, const FreeListSecrets &secrets) {
LIBC_ASSERT(!block.used() &&
"only free blocks can be placed on free lists");
- LIBC_ASSERT(block.inner_size_free() >= sizeof(FreeList) &&
+ LIBC_ASSERT(block.inner_size_free() >= sizeof(Node) &&
"block too small to accomodate free list node");
- push(new (block.usable_space()) Node);
+ push(new (block.usable_space()) Node, secrets);
}
/// Push an already-constructed node to the back of the list.
/// This allows pushing derived node types with additional data.
- LIBC_INLINE void push(Node *node) {
+ LIBC_INLINE void push(Node *node, const FreeListSecrets &secrets) {
if (begin_) {
- LIBC_ASSERT(BlockRef::from_usable_space(node).outer_size() ==
- begin_->block().outer_size() &&
- "freelist entries must have the same size");
- // Since the list is circular, insert the node immediately before begin_.
- node->prev = begin_->prev;
- node->next = begin_;
- begin_->prev->next = node;
- begin_->prev = node;
+ Node *begin_prev = secrets.decrypt_prev(begin_, begin_->prev);
+
+ LIBC_HARDENING_ASSERT(secrets.decrypt_next(begin_prev->next) == begin_ &&
+ "Corrupted free list links (push check)");
+
+ node->prev = secrets.encrypt_prev(node, begin_prev);
+ node->next = secrets.encrypt_next(begin_);
+ begin_prev->next = secrets.encrypt_next(node);
+ begin_->prev = secrets.encrypt_prev(begin_, node);
} else {
- begin_ = node->prev = node->next = node;
+ begin_ = node;
+ node->next = secrets.encrypt_next(node);
+ node->prev = secrets.encrypt_prev(node, node);
}
}
/// Pop the first node from the list.
- LIBC_INLINE void pop() { remove(begin_); }
+ LIBC_INLINE void pop(const FreeListSecrets &secrets) {
+ remove(begin_, secrets);
+ }
/// Remove an arbitrary node from the list.
- LIBC_INLINE void remove(Node *node) {
+ LIBC_INLINE void remove(Node *node, const FreeListSecrets &secrets) {
LIBC_ASSERT(begin_ && "cannot remove from empty list");
- if (node == node->next) {
+ Node *node_next = secrets.decrypt_next(node->next);
+ if (node == node_next) {
LIBC_ASSERT(node == begin_ &&
"a self-referential node must be the only element");
begin_ = nullptr;
} else {
- node->prev->next = node->next;
- node->next->prev = node->prev;
+ Node *node_prev = secrets.decrypt_prev(node, node->prev);
+
+ LIBC_HARDENING_ASSERT(
+ secrets.decrypt_next(node_prev->next) == node &&
+ "Corrupted free list links (remove check prev->next)");
+ LIBC_HARDENING_ASSERT(
+ secrets.decrypt_prev(node_next, node_next->prev) == node &&
+ "Corrupted free list links (remove check next->prev)");
+
+ node_prev->next = secrets.encrypt_next(node_next);
+ node_next->prev = secrets.encrypt_prev(node_next, node_prev);
if (begin_ == node)
- begin_ = node->next;
+ begin_ = node_next;
}
}
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index c05bf806990c8..f5e7700f0c606 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -37,12 +37,35 @@ using cpp::span;
LIBC_INLINE constexpr bool IsPow2(size_t x) { return x && (x & (x - 1)) == 0; }
+#ifndef LIBC_COPT_FREELIST_KEY0
+#define LIBC_COPT_FREELIST_KEY0 0
+#endif
+#ifndef LIBC_COPT_FREELIST_KEY1
+#define LIBC_COPT_FREELIST_KEY1 0
+#endif
+#ifndef LIBC_COPT_FREELIST_KEY2
+#define LIBC_COPT_FREELIST_KEY2 0
+#endif
+
class FreeListHeap {
public:
+#if LIBC_COPT_HARDEN_FREELIST
+ constexpr FreeListHeap(const FreeListSecrets &secrets =
+ {LIBC_COPT_FREELIST_KEY0, LIBC_COPT_FREELIST_KEY1,
+ LIBC_COPT_FREELIST_KEY2})
+ : begin(&_end), end(&__llvm_libc_heap_limit), secrets(secrets) {}
+
+ constexpr FreeListHeap(span<cpp::byte> region,
+ const FreeListSecrets &secrets =
+ {LIBC_COPT_FREELIST_KEY0, LIBC_COPT_FREELIST_KEY1,
+ LIBC_COPT_FREELIST_KEY2})
+ : begin(region.begin()), end(region.end()), secrets(secrets) {}
+#else
constexpr FreeListHeap() : begin(&_end), end(&__llvm_libc_heap_limit) {}
constexpr FreeListHeap(span<cpp::byte> region)
: begin(region.begin()), end(region.end()) {}
+#endif
void *allocate(size_t size);
void *aligned_allocate(size_t alignment, size_t size);
@@ -71,21 +94,39 @@ class FreeListHeap {
cpp::byte *end;
bool is_initialized = false;
FreeStore free_store;
+#if LIBC_COPT_HARDEN_FREELIST
+ const FreeListSecrets secrets;
+#endif
};
template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
public:
+#if LIBC_COPT_HARDEN_FREELIST
+ constexpr FreeListHeapBuffer(
+ const FreeListSecrets &secrets = {LIBC_COPT_FREELIST_KEY0,
+ LIBC_COPT_FREELIST_KEY1,
+ LIBC_COPT_FREELIST_KEY2})
+ : FreeListHeap{buffer, secrets}, buffer{} {}
+#else
constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
+#endif
private:
cpp::byte buffer[BUFF_SIZE];
};
+#if LIBC_COPT_HARDEN_FREELIST
+#define HEAP_SECRETS secrets
+#else
+#define HEAP_SECRETS \
+ FreeListSecrets {}
+#endif
+
[[gnu::noinline]] LIBC_INLINE void FreeListHeap::init() {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
auto result = BlockRef::init(region());
BlockRef block = *result;
- free_store.insert(block);
+ free_store.insert(block, HEAP_SECRETS);
is_initialized = true;
}
@@ -101,15 +142,15 @@ FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (!request_size)
return nullptr;
- BlockRef block = free_store.remove_best_fit(request_size);
+ BlockRef block = free_store.remove_best_fit(request_size, HEAP_SECRETS);
if (!block)
return nullptr;
auto block_info = BlockRef::allocate(block, alignment, size);
if (block_info.next)
- free_store.insert(block_info.next);
+ free_store.insert(block_info.next, HEAP_SECRETS);
if (block_info.prev)
- free_store.insert(block_info.prev);
+ free_store.insert(block_info.prev, HEAP_SECRETS);
block_info.block.mark_used();
return block_info.block.usable_space();
@@ -154,16 +195,16 @@ FreeListHeap::aligned_allocate(size_t alignment, size_t size) {
if (prev_free) {
// Remove from free store and merge.
- free_store.remove(prev_free);
+ free_store.remove(prev_free, HEAP_SECRETS);
block = prev_free;
block.merge_next();
}
if (!next.used()) {
- free_store.remove(next);
+ free_store.remove(next, HEAP_SECRETS);
block.merge_next();
}
// Add back to the freelist
- free_store.insert(block);
+ free_store.insert(block, HEAP_SECRETS);
}
[[gnu::noinline]] LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block,
@@ -184,10 +225,10 @@ FreeListHeap::aligned_allocate(size_t alignment, size_t size) {
// to be non-null.
LIBC_ASSERT(right && "right block must be non-null");
if (!right.used()) {
- free_store.remove(right);
+ free_store.remove(right, HEAP_SECRETS);
next_block.merge_next();
}
- free_store.insert(next_block);
+ free_store.insert(next_block, HEAP_SECRETS);
}
return true;
}
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 13b79c0debf39..9b440aaac7889 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -121,12 +121,14 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
LIBC_INLINE TLSFFreeStoreImpl &
operator=(const TLSFFreeStoreImpl &other) = delete;
- LIBC_INLINE void insert(BlockRef block);
- LIBC_INLINE void remove(BlockRef block);
- LIBC_INLINE BlockRef remove_best_fit(size_t size) {
- return find_and_remove_fit(size);
+ LIBC_INLINE void insert(BlockRef block, const FreeListSecrets &secrets);
+ LIBC_INLINE void remove(BlockRef block, const FreeListSecrets &secrets);
+ LIBC_INLINE BlockRef remove_best_fit(size_t size,
+ const FreeListSecrets &secrets) {
+ return find_and_remove_fit(size, secrets);
}
- LIBC_INLINE BlockRef find_and_remove_fit(size_t size);
+ LIBC_INLINE BlockRef find_and_remove_fit(size_t size,
+ const FreeListSecrets &secrets);
protected:
LIBC_INLINE static bool too_small(BlockRef block) {
@@ -148,7 +150,8 @@ template <typename CONFIG> class TLSFFreeStoreImpl {
LIBC_INLINE void clear_bit(size_t bit_index);
LIBC_INLINE bool get_bit(size_t bit_index) const;
LIBC_INLINE size_t find_first_bit_set_after(size_t bit_index) const;
- LIBC_INLINE BlockRef remove_first_fit_in_list(size_t index, size_t size);
+ LIBC_INLINE BlockRef remove_first_fit_in_list(size_t index, size_t size,
+ const FreeListSecrets &secrets);
LIBC_INLINE FreeTrie get_trie();
LIBC_INLINE BlockRef find_and_remove_fit_in_trie(size_t size);
LIBC_INLINE BlockRef pop_min_in_trie();
@@ -245,7 +248,9 @@ LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::pop_min_in_trie() {
}
template <typename CONFIG>
-LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
+LIBC_INLINE void
+TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block,
+ const FreeListSecrets &secrets) {
if (too_small(block))
return;
size_t bit_index = size_to_bit_index(block.inner_size());
@@ -257,12 +262,14 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::insert(BlockRef block) {
return;
}
- free_lists[bit_index].list.push(block);
+ free_lists[bit_index].list.push(block, secrets);
set_bit(bit_index);
}
template <typename CONFIG>
-LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
+LIBC_INLINE void
+TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block,
+ const FreeListSecrets &secrets) {
if (too_small(block))
return;
size_t bit_index = size_to_bit_index(block.inner_size());
@@ -277,14 +284,14 @@ LIBC_INLINE void TLSFFreeStoreImpl<CONFIG>::remove(BlockRef block) {
}
free_lists[bit_index].list.remove(
- reinterpret_cast<FreeList::Node *>(block.usable_space()));
+ reinterpret_cast<FreeList::Node *>(block.usable_space()), secrets);
if (free_lists[bit_index].list.empty())
clear_bit(bit_index);
}
template <typename CONFIG>
-LIBC_INLINE BlockRef
-TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
+LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(
+ size_t index, size_t size, const FreeListSecrets &secrets) {
FreeList::Node *begin_node = free_lists[index].list.begin();
if (begin_node == nullptr)
return BlockRef();
@@ -292,27 +299,27 @@ TLSFFreeStoreImpl<CONFIG>::remove_first_fit_in_list(size_t index, size_t size) {
FreeList::Node *cur = begin_node;
do {
if (cur->size() >= size) {
- free_lists[index].list.remove(cur);
+ free_lists[index].list.remove(cur, secrets);
if (free_lists[index].list.empty())
clear_bit(index);
return cur->block();
}
- cur = cur->next_node();
+ cur = free_lists[index].list.next_node(cur, secrets);
} while (cur != begin_node);
return BlockRef();
}
template <typename CONFIG>
-LIBC_INLINE BlockRef
-TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
+LIBC_INLINE BlockRef TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(
+ size_t size, const FreeListSecrets &secrets) {
size_t bit_index = size_to_bit_index(size);
if (LIBC_UNLIKELY(bit_index >= TOTAL_BITS - 1)) {
if constexpr (USE_TRIE)
return find_and_remove_fit_in_trie(size);
else
- return remove_first_fit_in_list(TOTAL_BITS - 1, size);
+ return remove_first_fit_in_list(TOTAL_BITS - 1, size, secrets);
}
// 1. Try oversized bins (guaranteed fit, but larger).
@@ -324,7 +331,7 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
}
BlockRef block = free_lists[oversized_bit].list.front();
- free_lists[oversized_bit].list.pop();
+ free_lists[oversized_bit].list.pop(secrets);
if (free_lists[oversized_bit].list.empty())
clear_bit(oversized_bit);
return block;
@@ -332,7 +339,7 @@ TLSFFreeStoreImpl<CONFIG>::find_and_remove_fit(size_t size) {
// 2. Try exact fit (fallback).
if (get_bit(bit_index)) {
- if (BlockRef block = remove_first_fit_in_list(bit_index, size))
+ if (BlockRef block = remove_first_fit_in_list(bit_index, size, secrets))
return block;
}
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 6c1b479ba8813..f9dfc64b3ad96 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -155,7 +155,7 @@ LIBC_INLINE void FreeTrie::push(BlockRef block) {
} else {
node->parent = nullptr;
}
- list.push(node);
+ list.push(node, FreeListSecrets{});
*cur = static_cast<Node *>(list.begin());
}
@@ -284,7 +284,7 @@ LIBC_INLINE FreeTrie::Node *FreeTrie::pop_min() {
LIBC_INLINE void FreeTrie::remove(Node *node) {
LIBC_ASSERT(!empty() && "cannot remove from empty trie");
FreeList list = node;
- list.pop();
+ list.pop(FreeListSecrets{});
Node *new_node = static_cast<Node *>(list.begin());
if (!new_node) {
// The freelist is empty. Replace the subtrie root with an arbitrary leaf.
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 68cc30152cd6d..34f96cc5fac33 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -32,9 +32,18 @@ using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::freelist_heap;
using LIBC_NAMESPACE::FreeListHeap;
using LIBC_NAMESPACE::FreeListHeapBuffer;
+using LIBC_NAMESPACE::FreeListSecrets;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::span;
+#if LIBC_COPT_HARDEN_FREELIST
+#define TEST_SECRETS_INIT FreeListSecrets{0x123, 0x456, 0x789}
+#define TEST_SECRETS_ARG , FreeListSecrets{0x123, 0x456, 0x789}
+#else
+#define TEST_SECRETS_INIT
+#define TEST_SECRETS_ARG
+#endif
+
// Similar to `LlvmLibcBlockTest` in block_test.cpp, we'd like to run the same
// tests independently for different parameters. In this case, we'd like to test
// functionality for a `FreeListHeap` and the global `freelist_heap` which was
@@ -49,16 +58,16 @@ using LIBC_NAMESPACE::cpp::span;
class LlvmLibcFreeListHeapTest##TestCase \
: public LIBC_NAMESPACE::testing::Test { \
public: \
- FreeListHeapBuffer<BufferSize> fake_global_buffer; \
+ FreeListHeapBuffer<BufferSize> fake_global_buffer{TEST_SECRETS_INIT}; \
void SetUp() override { \
- freelist_heap = \
- new (&fake_global_buffer) FreeListHeapBuffer<BufferSize>; \
+ freelist_heap = new (&fake_global_buffer) \
+ FreeListHeapBuffer<BufferSize>{TEST_SECRETS_INIT}; \
} \
void RunTest(FreeListHeap &allocator, [[maybe_unused]] size_t N); \
}; \
TEST_F(LlvmLibcFreeListHeapTest##TestCase, TestCase) { \
byte buf[BufferSize] = {byte(0)}; \
- FreeListHeap allocator(buf); \
+ FreeListHeap allocator(buf TEST_SECRETS_ARG); \
RunTest(allocator, BufferSize); \
RunTest(*freelist_heap, freelist_heap->region().size()); \
} \
@@ -112,7 +121,7 @@ TEST(LlvmLibcFreeListHeap, ReturnsNullWhenFull) {
constexpr size_t N = 2048;
byte buf[N];
- FreeListHeap allocator(buf);
+ FreeListHeap allocator(buf TEST_SECRETS_ARG);
bool went_null = false;
for (size_t i = 0; i < N; i++) {
@@ -308,7 +317,7 @@ TEST(LlvmLibcFreeListHeap, AlignedAllocUnalignedBuffer) {
byte buf[4096] = {byte(0)};
// Ensure the underlying buffer is poorly aligned.
- FreeListHeap allocator(span<byte>(buf).subspan(1));
+ FreeListHeap allocator(span<byte>(buf).subspan(1) TEST_SECRETS_ARG);
constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
diff --git a/libc/test/src/__support/freelist_test.cpp b/libc/test/src/__support/freelist_test.cpp
index 580f59ae62bd8..17b96b4d50380 100644
--- a/libc/test/src/__support/freelist_test.cpp
+++ b/libc/test/src/__support/freelist_test.cpp
@@ -18,9 +18,17 @@
using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::FreeList;
+using LIBC_NAMESPACE::FreeListSecrets;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::optional;
+#if LIBC_COPT_HARDEN_FREELIST
+#define TEST_SECRETS FreeListSecrets{0x1234, 0x5678, 0x9abc}
+#else
+#define TEST_SECRETS \
+ FreeListSecrets {}
+#endif
+
TEST(LlvmLibcFreeList, FreeList) {
byte mem[1024];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
@@ -35,24 +43,83 @@ TEST(LlvmLibcFreeList, FreeList) {
ASSERT_TRUE(maybeBlock.has_value());
FreeList list;
- list.push(block1);
+ list.push(block1, TEST_SECRETS);
ASSERT_FALSE(list.empty());
EXPECT_EQ(list.front().addr(), block1.addr());
- list.push(block2);
+ list.push(block2, TEST_SECRETS);
EXPECT_EQ(list.front().addr(), block1.addr());
- list.pop();
+ list.pop(TEST_SECRETS);
ASSERT_FALSE(list.empty());
EXPECT_EQ(list.front().addr(), block2.addr());
- list.pop();
+ list.pop(TEST_SECRETS);
ASSERT_TRUE(list.empty());
- list.push(block1);
- list.push(block2);
- list.remove(reinterpret_cast<FreeList::Node *>(block2.usable_space()));
+ list.push(block1, TEST_SECRETS);
+ list.push(block2, TEST_SECRETS);
+ list.remove(reinterpret_cast<FreeList::Node *>(block2.usable_space()),
+ TEST_SECRETS);
EXPECT_EQ(list.front().addr(), block1.addr());
- list.pop();
+ list.pop(TEST_SECRETS);
ASSERT_TRUE(list.empty());
}
+
+#if LIBC_COPT_HARDEN_FREELIST
+TEST(LlvmLibcFreeList, HardenedCorruptNext) {
+ byte mem[1024];
+ optional<BlockRef> maybeBlock = BlockRef::init(mem);
+ ASSERT_TRUE(maybeBlock.has_value());
+ BlockRef block1 = *maybeBlock;
+
+ maybeBlock = block1.split(128);
+ ASSERT_TRUE(maybeBlock.has_value());
+ BlockRef block2 = *maybeBlock;
+
+ FreeList list;
+ list.push(block1, TEST_SECRETS);
+ list.push(block2, TEST_SECRETS);
+
+ struct RawNode {
+ void *prev;
+ void *next;
+ };
+ RawNode *raw_node2 = reinterpret_cast<RawNode *>(block2.usable_space());
+ raw_node2->next = reinterpret_cast<void *>(0xDEADBEEF); // Corrupt next
+
+ EXPECT_DEATH(
+ [&] {
+ list.pop(TEST_SECRETS); // Should trap due to corrupted block2->next
+ },
+ WITH_SIGNAL(-1));
+}
+
+TEST(LlvmLibcFreeList, HardenedCorruptPrev) {
+ byte mem[1024];
+ optional<BlockRef> maybeBlock = BlockRef::init(mem);
+ ASSERT_TRUE(maybeBlock.has_value());
+ BlockRef block1 = *maybeBlock;
+
+ maybeBlock = block1.split(128);
+ ASSERT_TRUE(maybeBlock.has_value());
+ BlockRef block2 = *maybeBlock;
+
+ FreeList list;
+ list.push(block1, TEST_SECRETS);
+ list.push(block2, TEST_SECRETS);
+
+ struct RawNode {
+ void *prev;
+ void *next;
+ };
+ RawNode *raw_node2 = reinterpret_cast<RawNode *>(block2.usable_space());
+ raw_node2->prev = reinterpret_cast<void *>(0xDEADBEEF); // Corrupt prev
+
+ EXPECT_DEATH(
+ [&] {
+ list.pop(TEST_SECRETS); // Should trap due to corrupted block2->prev
+ },
+ WITH_SIGNAL(-1));
+}
+#endif
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 623d9f0858eac..83115da15f4e4 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -16281,6 +16281,7 @@ libc_support_library(
":__support_common",
":__support_libc_assert",
":__support_macros_config",
+ ":hdr_stdint_proxy",
],
)
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
index 0698f45ceabc9..7a221e62bfee3 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
@@ -121,3 +121,53 @@ libc_test(
"//libc:__support_macros_properties_types",
],
)
+
+libc_test(
+ name = "freelist_test",
+ srcs = ["freelist_test.cpp"],
+ deps = [
+ "//libc:__support_block",
+ "//libc:__support_cpp_optional",
+ "//libc:freelist",
+ ],
+)
+
+libc_test(
+ name = "freelist_hardened_test",
+ srcs = ["freelist_test.cpp"],
+ defines = ["LIBC_COPT_HARDEN_FREELIST=1"],
+ deps = [
+ "//libc:__support_block",
+ "//libc:__support_cpp_optional",
+ "//libc:freelist",
+ ],
+)
+
+libc_test(
+ name = "freelist_heap_test",
+ srcs = ["freelist_heap_test.cpp"],
+ deps = [
+ "//libc:__support_block",
+ "//libc:__support_cpp_optional",
+ "//libc:__support_cpp_span",
+ "//libc:freelist_heap",
+ "//libc:memcmp",
+ "//libc:memcpy",
+ "//libc:string_memory_utils",
+ ],
+)
+
+libc_test(
+ name = "freelist_heap_hardened_test",
+ srcs = ["freelist_heap_test.cpp"],
+ defines = ["LIBC_COPT_HARDEN_FREELIST=1"],
+ deps = [
+ "//libc:__support_block",
+ "//libc:__support_cpp_optional",
+ "//libc:__support_cpp_span",
+ "//libc:freelist_heap",
+ "//libc:memcmp",
+ "//libc:memcpy",
+ "//libc:string_memory_utils",
+ ],
+)
More information about the libc-commits
mailing list