[libc-commits] [libc] [libc] add option for hardened freelist (PR #205382)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Wed Jul 1 12:03:53 PDT 2026
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/205382
>From 6758e2f895803c21144a4058f74ac7f8d7725842 Mon Sep 17 00:00:00 2001
From: yfzhu <yfzhu at google.com>
Date: Tue, 23 Jun 2026 09:17:39 -0700
Subject: [PATCH 1/2] [libc] add option for hardened freelist
This change adds the LIBC_COPT_HARDEN_FREELIST option which enables
encryption of forward (next) and backward (prev) pointers in the baremetal
freelist heap. When enabled, the FreeListHeap class stores three const
uintptr_t keys, which are initialized in its constructors. These keys are
passed down to FreeStore and FreeList operations to perform encoding and
corruption verification. If corruption is detected, the allocator will trap.
TAG=agy
CONV=4118a58d-30c6-4fd4-8e45-defe707d8bba
---
libc/src/__support/CMakeLists.txt | 1 +
libc/src/__support/freelist.cpp | 43 +++++---
libc/src/__support/freelist.h | 100 ++++++++++++++++--
libc/src/__support/freelist_heap.h | 41 ++++---
libc/src/__support/freestore.h | 21 ++--
libc/src/__support/freetrie.cpp | 2 +-
libc/src/__support/freetrie.h | 2 +-
libc/src/__support/macros/attributes.h | 12 +++
.../test/src/__support/freelist_heap_test.cpp | 12 +++
libc/test/src/__support/freelist_test.cpp | 83 +++++++++++++--
libc/test/src/__support/freestore_test.cpp | 47 ++++----
11 files changed, 290 insertions(+), 74 deletions(-)
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index f4cb283976bc3..19d0d95873812 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -35,6 +35,7 @@ add_object_library(
.block
libc.src.__support.fixedvector
libc.src.__support.CPP.array
+ libc.src.__support.CPP.bit
libc.src.__support.CPP.cstddef
libc.src.__support.CPP.new
libc.src.__support.CPP.span
diff --git a/libc/src/__support/freelist.cpp b/libc/src/__support/freelist.cpp
index 28f73b2726d7d..600247c94e029 100644
--- a/libc/src/__support/freelist.cpp
+++ b/libc/src/__support/freelist.cpp
@@ -15,32 +15,45 @@
namespace LIBC_NAMESPACE_DECL {
-void FreeList::push(Node *node) {
+void FreeList::push(Node *node, const FreeListSecrets &secrets) {
if (begin_) {
- LIBC_ASSERT(BlockRef::from_usable_space(node).outer_size() ==
- begin_->block().outer_size() &&
- "freelist entries must have the same size");
- // Since the list is circular, insert the node immediately before begin_.
- node->prev = begin_->prev;
- node->next = begin_;
- begin_->prev->next = node;
- begin_->prev = node;
+ Node *begin_prev = secrets.decrypt_prev(begin_, begin_->prev);
+
+ LIBC_HARDENING_ASSERT(secrets.decrypt_next(begin_prev->next) == begin_ &&
+ "Corrupted free list links (push check)");
+
+ node->prev = secrets.encrypt_prev(node, begin_prev);
+ node->next = secrets.encrypt_next(begin_);
+ begin_prev->next = secrets.encrypt_next(node);
+ begin_->prev = secrets.encrypt_prev(begin_, node);
} else {
- begin_ = node->prev = node->next = node;
+ begin_ = node;
+ node->next = secrets.encrypt_next(node);
+ node->prev = secrets.encrypt_prev(node, node);
}
}
-void FreeList::remove(Node *node) {
+void FreeList::remove(Node *node, const FreeListSecrets &secrets) {
LIBC_ASSERT(begin_ && "cannot remove from empty list");
- if (node == node->next) {
+ Node *node_next = secrets.decrypt_next(node->next);
+ if (node == node_next) {
LIBC_ASSERT(node == begin_ &&
"a self-referential node must be the only element");
begin_ = nullptr;
} else {
- node->prev->next = node->next;
- node->next->prev = node->prev;
+ Node *node_prev = secrets.decrypt_prev(node, node->prev);
+
+ LIBC_HARDENING_ASSERT(
+ secrets.decrypt_next(node_prev->next) == node &&
+ "Corrupted free list links (remove check prev->next)");
+ LIBC_HARDENING_ASSERT(
+ secrets.decrypt_prev(node_next, node_next->prev) == node &&
+ "Corrupted free list links (remove check next->prev)");
+
+ node_prev->next = secrets.encrypt_next(node_next);
+ node_next->prev = secrets.encrypt_prev(node_next, node_prev);
if (begin_ == node)
- begin_ = node->next;
+ begin_ = node_next;
}
}
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index 48e70c7c29df6..2d65b450cbd47 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -15,9 +15,85 @@
#define LLVM_LIBC_SRC___SUPPORT_FREELIST_H
#include "block.h"
+#include "hdr/stdint_proxy.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/libc_assert.h"
+#include "src/__support/macros/config.h"
+
+#ifndef LIBC_COPT_HARDEN_FREELIST
+#define LIBC_COPT_HARDEN_FREELIST false
+#endif
+
+#if LIBC_COPT_HARDEN_FREELIST
+#define LIBC_HARDENING_ASSERT(cond) \
+ do { \
+ if (LIBC_UNLIKELY(!(cond))) { \
+ __builtin_trap(); \
+ } \
+ } while (0)
+#else
+#define LIBC_HARDENING_ASSERT(cond) LIBC_ASSERT(cond)
+#endif
namespace LIBC_NAMESPACE_DECL {
+struct FreeListSecrets {
+ static constexpr int NODE_PTR_ROTATE_DISTANCE = 17;
+
+#if LIBC_COPT_HARDEN_FREELIST
+ uintptr_t k0;
+ uintptr_t k1;
+ uintptr_t k2;
+
+ LIBC_INLINE constexpr FreeListSecrets(uintptr_t k0, uintptr_t k1,
+ uintptr_t k2)
+ : k0(k0), k1(k1), k2(k2) {}
+#else
+ LIBC_INLINE constexpr FreeListSecrets() = default;
+#endif
+
+ template <typename T> LIBC_INLINE T *decrypt_next(T *next_val) const {
+#if LIBC_COPT_HARDEN_FREELIST
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(next_val) ^ k0);
+#else
+ return next_val;
+#endif
+ }
+
+ template <typename T>
+ LIBC_INLINE T *decrypt_prev([[maybe_unused]] const void *node,
+ T *prev_val) const {
+#if LIBC_COPT_HARDEN_FREELIST
+ uintptr_t val = reinterpret_cast<uintptr_t>(prev_val) ^ k2 ^
+ reinterpret_cast<uintptr_t>(node);
+ val = cpp::rotl(val, NODE_PTR_ROTATE_DISTANCE);
+ return reinterpret_cast<T *>(val ^ k1);
+#else
+ return prev_val;
+#endif
+ }
+
+ template <typename T> LIBC_INLINE T *encrypt_next(T *next_val) const {
+#if LIBC_COPT_HARDEN_FREELIST
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(next_val) ^ k0);
+#else
+ return next_val;
+#endif
+ }
+
+ template <typename T>
+ LIBC_INLINE T *encrypt_prev([[maybe_unused]] const void *node,
+ T *prev_val) const {
+#if LIBC_COPT_HARDEN_FREELIST
+ uintptr_t val = reinterpret_cast<uintptr_t>(prev_val) ^ k1;
+ val = cpp::rotr(val, NODE_PTR_ROTATE_DISTANCE);
+ return reinterpret_cast<T *>(val ^ reinterpret_cast<uintptr_t>(node) ^ k2);
+#else
+ return prev_val;
+#endif
+ }
+};
+
/// A circularly-linked FIFO list storing free Blocks. All Blocks on a list
/// are the same size. The blocks are referenced by Nodes in the list; the list
/// refers to these, but it does not own them.
@@ -65,25 +141,37 @@ class FreeList {
/// @returns The first block in the list.
LIBC_INLINE BlockRef front() { return begin_->block(); }
+ LIBC_INLINE Node *next_node(const Node *node,
+ const FreeListSecrets &secrets) const {
+ return node ? secrets.decrypt_next(node->next) : nullptr;
+ }
+
+ LIBC_INLINE Node *prev_node(const Node *node,
+ const FreeListSecrets &secrets) const {
+ return node ? secrets.decrypt_prev(node, node->prev) : nullptr;
+ }
+
/// Push a block to the back of the list.
/// The block must be large enough to contain a node.
- LIBC_INLINE void push(BlockRef block) {
+ LIBC_INLINE void push(BlockRef block, const FreeListSecrets &secrets) {
LIBC_ASSERT(!block.used() &&
"only free blocks can be placed on free lists");
- LIBC_ASSERT(block.inner_size_free() >= sizeof(FreeList) &&
+ LIBC_ASSERT(block.inner_size_free() >= sizeof(Node) &&
"block too small to accomodate free list node");
- push(new (block.usable_space()) Node);
+ push(new (block.usable_space()) Node, secrets);
}
/// Push an already-constructed node to the back of the list.
/// This allows pushing derived node types with additional data.
- void push(Node *node);
+ void push(Node *node, const FreeListSecrets &secrets);
/// Pop the first node from the list.
- LIBC_INLINE void pop() { remove(begin_); }
+ LIBC_INLINE void pop(const FreeListSecrets &secrets) {
+ remove(begin_, secrets);
+ }
/// Remove an arbitrary node from the list.
- void remove(Node *node);
+ void remove(Node *node, const FreeListSecrets &secrets);
private:
Node *begin_;
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index a591d63ddd4e5..11dd8e2fe6801 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -21,6 +21,7 @@
#include "src/__support/CPP/optional.h"
#include "src/__support/CPP/span.h"
#include "src/__support/libc_assert.h"
+#include "src/__support/macros/attributes.h"
#include "src/__support/macros/config.h"
#include "src/__support/math_extras.h"
#include "src/string/memory_utils/inline_memcpy.h"
@@ -53,9 +54,13 @@ class FreeListHeap {
cpp::span<cpp::byte> region() const { return {begin, end}; }
-private:
- void init();
+#if LIBC_COPT_HARDEN_FREELIST
+ void init(const FreeListSecrets &secrets);
+#else
+ void init(const FreeListSecrets &secrets = {});
+#endif
+private:
void *allocate_impl(size_t alignment, size_t size);
span<cpp::byte> block_to_span(BlockRef block) {
@@ -70,6 +75,7 @@ class FreeListHeap {
cpp::byte *end;
bool is_initialized = false;
FreeStore free_store;
+ LIBC_NO_UNIQUE_ADDRESS FreeListSecrets secrets;
};
template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
@@ -80,12 +86,14 @@ template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
cpp::byte buffer[BUFF_SIZE];
};
-LIBC_INLINE void FreeListHeap::init() {
+[[gnu::noinline]] LIBC_INLINE void
+FreeListHeap::init(const FreeListSecrets &secrets_arg) {
LIBC_ASSERT(!is_initialized && "duplicate initialization");
+ secrets = secrets_arg;
auto result = BlockRef::init(region());
BlockRef block = *result;
free_store.set_range({0, cpp::bit_ceil(block.inner_size())});
- free_store.insert(block);
+ free_store.insert(block, secrets);
is_initialized = true;
}
@@ -93,22 +101,29 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (size == 0)
return nullptr;
- if (!is_initialized)
+ if (!is_initialized) {
+#if LIBC_COPT_HARDEN_FREELIST
+ LIBC_HARDENING_ASSERT(
+ false &&
+ "Hardened heap must be explicitly initialized via init(secrets)");
+#else
init();
+#endif
+ }
size_t request_size = BlockRef::min_size_for_allocation(alignment, size);
if (!request_size)
return nullptr;
- BlockRef block = free_store.remove_best_fit(request_size);
+ BlockRef block = free_store.remove_best_fit(request_size, secrets);
if (!block)
return nullptr;
auto block_info = BlockRef::allocate(block, alignment, size);
if (block_info.next)
- free_store.insert(block_info.next);
+ free_store.insert(block_info.next, secrets);
if (block_info.prev)
- free_store.insert(block_info.prev);
+ free_store.insert(block_info.prev, secrets);
block_info.block.mark_used();
return block_info.block.usable_space();
@@ -153,16 +168,16 @@ LIBC_INLINE void FreeListHeap::free(void *ptr) {
if (prev_free) {
// Remove from free store and merge.
- free_store.remove(prev_free);
+ free_store.remove(prev_free, secrets);
block = prev_free;
block.merge_next();
}
if (!next.used()) {
- free_store.remove(next);
+ free_store.remove(next, secrets);
block.merge_next();
}
// Add back to the freelist
- free_store.insert(block);
+ free_store.insert(block, secrets);
}
LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
@@ -182,10 +197,10 @@ LIBC_INLINE bool FreeListHeap::shrink_in_place(BlockRef block, size_t size) {
// to be non-null.
LIBC_ASSERT(right && "right block must be non-null");
if (!right.used()) {
- free_store.remove(right);
+ free_store.remove(right, secrets);
next_block.merge_next();
}
- free_store.insert(next_block);
+ free_store.insert(next_block, secrets);
}
return true;
}
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index adc0e061ace93..f981ded370998 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -36,15 +36,15 @@ class FreeStore {
/// Insert a free block. If the block is too small to be tracked, nothing
/// happens.
- void insert(BlockRef block);
+ void insert(BlockRef block, const FreeListSecrets &secrets);
/// Remove a free block. If the block is too small to be tracked, nothing
/// happens.
- void remove(BlockRef block);
+ void remove(BlockRef block, const FreeListSecrets &secrets);
/// Remove a best-fit free block that can contain the given size when
/// allocated. Returns nullptr if there is no such block.
- BlockRef remove_best_fit(size_t size);
+ BlockRef remove_best_fit(size_t size, const FreeListSecrets &secrets);
private:
static constexpr size_t MIN_OUTER_SIZE = align_up(
@@ -68,30 +68,33 @@ class FreeStore {
FreeTrie large_trie;
};
-LIBC_INLINE void FreeStore::insert(BlockRef block) {
+LIBC_INLINE void FreeStore::insert(BlockRef block,
+ const FreeListSecrets &secrets) {
if (too_small(block))
return;
if (is_small(block))
- small_list(block).push(block);
+ small_list(block).push(block, secrets);
else
large_trie.push(block);
}
-LIBC_INLINE void FreeStore::remove(BlockRef block) {
+LIBC_INLINE void FreeStore::remove(BlockRef block,
+ const FreeListSecrets &secrets) {
if (too_small(block))
return;
if (is_small(block)) {
small_list(block).remove(
- reinterpret_cast<FreeList::Node *>(block.usable_space()));
+ reinterpret_cast<FreeList::Node *>(block.usable_space()), secrets);
} else {
large_trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
}
}
-LIBC_INLINE BlockRef FreeStore::remove_best_fit(size_t size) {
+LIBC_INLINE BlockRef
+FreeStore::remove_best_fit(size_t size, const FreeListSecrets &secrets) {
if (FreeList *list = find_best_small_fit(size)) {
BlockRef block = list->front();
- list->pop();
+ list->pop(secrets);
return block;
}
if (FreeTrie::Node *best_fit = large_trie.find_best_fit(size)) {
diff --git a/libc/src/__support/freetrie.cpp b/libc/src/__support/freetrie.cpp
index e76efe717f215..3dc93adf7ccf4 100644
--- a/libc/src/__support/freetrie.cpp
+++ b/libc/src/__support/freetrie.cpp
@@ -13,7 +13,7 @@ namespace LIBC_NAMESPACE_DECL {
void FreeTrie::remove(Node *node) {
LIBC_ASSERT(!empty() && "cannot remove from empty trie");
FreeList list = node;
- list.pop();
+ list.pop(FreeListSecrets{});
Node *new_node = static_cast<Node *>(list.begin());
if (!new_node) {
// The freelist is empty. Replace the subtrie root with an arbitrary leaf.
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 9e35463462b38..ccc8c43836cfe 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -152,7 +152,7 @@ LIBC_INLINE void FreeTrie::push(BlockRef block) {
} else {
node->parent = nullptr;
}
- list.push(node);
+ list.push(node, FreeListSecrets{});
*cur = static_cast<Node *>(list.begin());
}
diff --git a/libc/src/__support/macros/attributes.h b/libc/src/__support/macros/attributes.h
index cffcabcd29bd3..64012b415f235 100644
--- a/libc/src/__support/macros/attributes.h
+++ b/libc/src/__support/macros/attributes.h
@@ -165,4 +165,16 @@ LIBC_THREAD_MODE_EXTERNAL.
#define LIBC_NO_SANITIZE_OOB_ACCESS
#endif
+#if defined(__has_cpp_attribute)
+#if __has_cpp_attribute(msvc::no_unique_address)
+#define LIBC_NO_UNIQUE_ADDRESS [[msvc::no_unique_address]]
+#elif __has_cpp_attribute(no_unique_address)
+#define LIBC_NO_UNIQUE_ADDRESS [[no_unique_address]]
+#endif
+#endif
+
+#ifndef LIBC_NO_UNIQUE_ADDRESS
+#define LIBC_NO_UNIQUE_ADDRESS
+#endif
+
#endif // LLVM_LIBC_SRC___SUPPORT_MACROS_ATTRIBUTES_H
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 68cc30152cd6d..c4117c35049ec 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -32,9 +32,17 @@ using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::freelist_heap;
using LIBC_NAMESPACE::FreeListHeap;
using LIBC_NAMESPACE::FreeListHeapBuffer;
+using LIBC_NAMESPACE::FreeListSecrets;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::span;
+#if LIBC_COPT_HARDEN_FREELIST
+#define TEST_SECRETS_INIT FreeListSecrets{0x123, 0x456, 0x789}
+#else
+#define TEST_SECRETS_INIT \
+ FreeListSecrets {}
+#endif
+
// Similar to `LlvmLibcBlockTest` in block_test.cpp, we'd like to run the same
// tests independently for different parameters. In this case, we'd like to test
// functionality for a `FreeListHeap` and the global `freelist_heap` which was
@@ -53,12 +61,14 @@ using LIBC_NAMESPACE::cpp::span;
void SetUp() override { \
freelist_heap = \
new (&fake_global_buffer) FreeListHeapBuffer<BufferSize>; \
+ freelist_heap->init(TEST_SECRETS_INIT); \
} \
void RunTest(FreeListHeap &allocator, [[maybe_unused]] size_t N); \
}; \
TEST_F(LlvmLibcFreeListHeapTest##TestCase, TestCase) { \
byte buf[BufferSize] = {byte(0)}; \
FreeListHeap allocator(buf); \
+ allocator.init(TEST_SECRETS_INIT); \
RunTest(allocator, BufferSize); \
RunTest(*freelist_heap, freelist_heap->region().size()); \
} \
@@ -113,6 +123,7 @@ TEST(LlvmLibcFreeListHeap, ReturnsNullWhenFull) {
byte buf[N];
FreeListHeap allocator(buf);
+ allocator.init(TEST_SECRETS_INIT);
bool went_null = false;
for (size_t i = 0; i < N; i++) {
@@ -309,6 +320,7 @@ TEST(LlvmLibcFreeListHeap, AlignedAllocUnalignedBuffer) {
// Ensure the underlying buffer is poorly aligned.
FreeListHeap allocator(span<byte>(buf).subspan(1));
+ allocator.init(TEST_SECRETS_INIT);
constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
diff --git a/libc/test/src/__support/freelist_test.cpp b/libc/test/src/__support/freelist_test.cpp
index 580f59ae62bd8..17b96b4d50380 100644
--- a/libc/test/src/__support/freelist_test.cpp
+++ b/libc/test/src/__support/freelist_test.cpp
@@ -18,9 +18,17 @@
using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::FreeList;
+using LIBC_NAMESPACE::FreeListSecrets;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::optional;
+#if LIBC_COPT_HARDEN_FREELIST
+#define TEST_SECRETS FreeListSecrets{0x1234, 0x5678, 0x9abc}
+#else
+#define TEST_SECRETS \
+ FreeListSecrets {}
+#endif
+
TEST(LlvmLibcFreeList, FreeList) {
byte mem[1024];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
@@ -35,24 +43,83 @@ TEST(LlvmLibcFreeList, FreeList) {
ASSERT_TRUE(maybeBlock.has_value());
FreeList list;
- list.push(block1);
+ list.push(block1, TEST_SECRETS);
ASSERT_FALSE(list.empty());
EXPECT_EQ(list.front().addr(), block1.addr());
- list.push(block2);
+ list.push(block2, TEST_SECRETS);
EXPECT_EQ(list.front().addr(), block1.addr());
- list.pop();
+ list.pop(TEST_SECRETS);
ASSERT_FALSE(list.empty());
EXPECT_EQ(list.front().addr(), block2.addr());
- list.pop();
+ list.pop(TEST_SECRETS);
ASSERT_TRUE(list.empty());
- list.push(block1);
- list.push(block2);
- list.remove(reinterpret_cast<FreeList::Node *>(block2.usable_space()));
+ list.push(block1, TEST_SECRETS);
+ list.push(block2, TEST_SECRETS);
+ list.remove(reinterpret_cast<FreeList::Node *>(block2.usable_space()),
+ TEST_SECRETS);
EXPECT_EQ(list.front().addr(), block1.addr());
- list.pop();
+ list.pop(TEST_SECRETS);
ASSERT_TRUE(list.empty());
}
+
+#if LIBC_COPT_HARDEN_FREELIST
+TEST(LlvmLibcFreeList, HardenedCorruptNext) {
+ byte mem[1024];
+ optional<BlockRef> maybeBlock = BlockRef::init(mem);
+ ASSERT_TRUE(maybeBlock.has_value());
+ BlockRef block1 = *maybeBlock;
+
+ maybeBlock = block1.split(128);
+ ASSERT_TRUE(maybeBlock.has_value());
+ BlockRef block2 = *maybeBlock;
+
+ FreeList list;
+ list.push(block1, TEST_SECRETS);
+ list.push(block2, TEST_SECRETS);
+
+ struct RawNode {
+ void *prev;
+ void *next;
+ };
+ RawNode *raw_node2 = reinterpret_cast<RawNode *>(block2.usable_space());
+ raw_node2->next = reinterpret_cast<void *>(0xDEADBEEF); // Corrupt next
+
+ EXPECT_DEATH(
+ [&] {
+ list.pop(TEST_SECRETS); // Should trap due to corrupted block2->next
+ },
+ WITH_SIGNAL(-1));
+}
+
+TEST(LlvmLibcFreeList, HardenedCorruptPrev) {
+ byte mem[1024];
+ optional<BlockRef> maybeBlock = BlockRef::init(mem);
+ ASSERT_TRUE(maybeBlock.has_value());
+ BlockRef block1 = *maybeBlock;
+
+ maybeBlock = block1.split(128);
+ ASSERT_TRUE(maybeBlock.has_value());
+ BlockRef block2 = *maybeBlock;
+
+ FreeList list;
+ list.push(block1, TEST_SECRETS);
+ list.push(block2, TEST_SECRETS);
+
+ struct RawNode {
+ void *prev;
+ void *next;
+ };
+ RawNode *raw_node2 = reinterpret_cast<RawNode *>(block2.usable_space());
+ raw_node2->prev = reinterpret_cast<void *>(0xDEADBEEF); // Corrupt prev
+
+ EXPECT_DEATH(
+ [&] {
+ list.pop(TEST_SECRETS); // Should trap due to corrupted block2->prev
+ },
+ WITH_SIGNAL(-1));
+}
+#endif
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index 61103a9126c08..345f8295f374c 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -18,6 +18,7 @@
using LIBC_NAMESPACE::BlockRef;
using LIBC_NAMESPACE::FreeList;
+using LIBC_NAMESPACE::FreeListSecrets;
using LIBC_NAMESPACE::FreeStore;
using LIBC_NAMESPACE::FreeTrie;
using LIBC_NAMESPACE::cpp::byte;
@@ -25,6 +26,7 @@ using LIBC_NAMESPACE::cpp::optional;
// Inserting or removing blocks too small to be tracked does nothing.
TEST(LlvmLibcFreeStore, TooSmall) {
+ FreeListSecrets secrets{};
byte mem[1024];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
@@ -39,15 +41,16 @@ TEST(LlvmLibcFreeStore, TooSmall) {
FreeStore store;
store.set_range({0, 4096});
- store.insert(too_small);
- store.insert(remainder);
+ store.insert(too_small, secrets);
+ store.insert(remainder, secrets);
- EXPECT_EQ(store.remove_best_fit(too_small.inner_size()).addr(),
+ EXPECT_EQ(store.remove_best_fit(too_small.inner_size(), secrets).addr(),
remainder.addr());
- store.remove(too_small);
+ store.remove(too_small, secrets);
}
TEST(LlvmLibcFreeStore, RemoveBestFit) {
+ FreeListSecrets secrets{};
byte mem[1024];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
@@ -69,34 +72,36 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) {
FreeStore store;
store.set_range({0, 4096});
- store.insert(smallest);
+ store.insert(smallest, secrets);
if (largest_small != smallest)
- store.insert(largest_small);
- store.insert(remainder);
+ store.insert(largest_small, secrets);
+ store.insert(remainder, secrets);
// Find exact match for smallest.
- ASSERT_EQ(store.remove_best_fit(smallest.inner_size()).addr(),
+ ASSERT_EQ(store.remove_best_fit(smallest.inner_size(), secrets).addr(),
smallest.addr());
- store.insert(smallest);
+ store.insert(smallest, secrets);
// Find exact match for largest.
- ASSERT_EQ(store.remove_best_fit(largest_small.inner_size()).addr(),
+ ASSERT_EQ(store.remove_best_fit(largest_small.inner_size(), secrets).addr(),
largest_small.addr());
- store.insert(largest_small);
+ store.insert(largest_small, secrets);
// Search small list for best fit.
BlockRef next_smallest =
largest_small == smallest ? remainder : largest_small;
- ASSERT_EQ(store.remove_best_fit(smallest.inner_size() + 1).addr(),
+ ASSERT_EQ(store.remove_best_fit(smallest.inner_size() + 1, secrets).addr(),
next_smallest.addr());
- store.insert(next_smallest);
+ store.insert(next_smallest, secrets);
// Continue search for best fit to large blocks.
- EXPECT_EQ(store.remove_best_fit(largest_small.inner_size() + 1).addr(),
- remainder.addr());
+ EXPECT_EQ(
+ store.remove_best_fit(largest_small.inner_size() + 1, secrets).addr(),
+ remainder.addr());
}
TEST(LlvmLibcFreeStore, Remove) {
+ FreeListSecrets secrets{};
byte mem[1024];
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
@@ -109,13 +114,13 @@ TEST(LlvmLibcFreeStore, Remove) {
FreeStore store;
store.set_range({0, 4096});
- store.insert(small);
- store.insert(remainder);
+ store.insert(small, secrets);
+ store.insert(remainder, secrets);
- store.remove(remainder);
- ASSERT_EQ(store.remove_best_fit(remainder.inner_size()).addr(),
+ store.remove(remainder, secrets);
+ ASSERT_EQ(store.remove_best_fit(remainder.inner_size(), secrets).addr(),
BlockRef().addr());
- store.remove(small);
- ASSERT_EQ(store.remove_best_fit(small.inner_size()).addr(),
+ store.remove(small, secrets);
+ ASSERT_EQ(store.remove_best_fit(small.inner_size(), secrets).addr(),
BlockRef().addr());
}
>From d8a7da5d9d27f30d8ba11181bed02ec160950df2 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 1 Jul 2026 11:42:04 -0700
Subject: [PATCH 2/2] [libc] add sanitize_heap to FreeListHeap
---
libc/src/__support/freelist.cpp | 18 +++++++++
libc/src/__support/freelist.h | 4 ++
libc/src/__support/freelist_heap.h | 8 ++++
libc/src/__support/freestore.h | 16 ++++++--
libc/src/__support/freetrie.cpp | 16 +++++++-
libc/src/__support/freetrie.h | 12 ++++--
.../test/src/__support/freelist_heap_test.cpp | 16 ++++++++
libc/test/src/__support/freetrie_test.cpp | 38 +++++++++++--------
8 files changed, 104 insertions(+), 24 deletions(-)
diff --git a/libc/src/__support/freelist.cpp b/libc/src/__support/freelist.cpp
index 600247c94e029..430f2cd818e48 100644
--- a/libc/src/__support/freelist.cpp
+++ b/libc/src/__support/freelist.cpp
@@ -57,4 +57,22 @@ void FreeList::remove(Node *node, const FreeListSecrets &secrets) {
}
}
+void FreeList::sanitize(const FreeListSecrets &secrets) const {
+ if (!begin_)
+ return;
+ Node *curr = begin_;
+ do {
+ Node *next_node = secrets.decrypt_next(curr->next);
+ Node *prev_node = secrets.decrypt_prev(curr, curr->prev);
+ (void)prev_node;
+ LIBC_HARDENING_ASSERT(
+ secrets.decrypt_next(prev_node->next) == curr &&
+ "Corrupted free list links (sanitize check prev->next)");
+ LIBC_HARDENING_ASSERT(
+ secrets.decrypt_prev(next_node, next_node->prev) == curr &&
+ "Corrupted free list links (sanitize check next->prev)");
+ curr = next_node;
+ } while (curr != begin_);
+}
+
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index 2d65b450cbd47..70e91305c4554 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -48,6 +48,7 @@ struct FreeListSecrets {
LIBC_INLINE constexpr FreeListSecrets(uintptr_t k0, uintptr_t k1,
uintptr_t k2)
: k0(k0), k1(k1), k2(k2) {}
+ LIBC_INLINE constexpr FreeListSecrets() : k0(0), k1(0), k2(0) {}
#else
LIBC_INLINE constexpr FreeListSecrets() = default;
#endif
@@ -173,6 +174,9 @@ class FreeList {
/// Remove an arbitrary node from the list.
void remove(Node *node, const FreeListSecrets &secrets);
+ /// Verify secret invariants for all nodes in the list.
+ void sanitize(const FreeListSecrets &secrets) const;
+
private:
Node *begin_;
};
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 11dd8e2fe6801..46fc0083a17b7 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -60,6 +60,8 @@ class FreeListHeap {
void init(const FreeListSecrets &secrets = {});
#endif
+ void sanitize_heap() const;
+
private:
void *allocate_impl(size_t alignment, size_t size);
@@ -254,6 +256,12 @@ LIBC_INLINE void *FreeListHeap::calloc(size_t num, size_t size) {
return ptr;
}
+LIBC_INLINE void FreeListHeap::sanitize_heap() const {
+ if (!is_initialized)
+ return;
+ free_store.sanitize(secrets);
+}
+
extern FreeListHeap *freelist_heap;
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index f981ded370998..808fcc48b0617 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -46,6 +46,9 @@ class FreeStore {
/// allocated. Returns nullptr if there is no such block.
BlockRef remove_best_fit(size_t size, const FreeListSecrets &secrets);
+ /// Verify secret invariants for all free lists in the store.
+ void sanitize(const FreeListSecrets &secrets) const;
+
private:
static constexpr size_t MIN_OUTER_SIZE = align_up(
BlockRef::HEADER_SIZE + sizeof(FreeList::Node), BlockRef::MIN_ALIGN);
@@ -75,7 +78,7 @@ LIBC_INLINE void FreeStore::insert(BlockRef block,
if (is_small(block))
small_list(block).push(block, secrets);
else
- large_trie.push(block);
+ large_trie.push(block, secrets);
}
LIBC_INLINE void FreeStore::remove(BlockRef block,
@@ -86,7 +89,8 @@ LIBC_INLINE void FreeStore::remove(BlockRef block,
small_list(block).remove(
reinterpret_cast<FreeList::Node *>(block.usable_space()), secrets);
} else {
- large_trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()));
+ large_trie.remove(reinterpret_cast<FreeTrie::Node *>(block.usable_space()),
+ secrets);
}
}
@@ -99,7 +103,7 @@ FreeStore::remove_best_fit(size_t size, const FreeListSecrets &secrets) {
}
if (FreeTrie::Node *best_fit = large_trie.find_best_fit(size)) {
BlockRef block = best_fit->block();
- large_trie.remove(best_fit);
+ large_trie.remove(best_fit, secrets);
return block;
}
return BlockRef();
@@ -118,6 +122,12 @@ LIBC_INLINE FreeList *FreeStore::find_best_small_fit(size_t size) {
return nullptr;
}
+LIBC_INLINE void FreeStore::sanitize(const FreeListSecrets &secrets) const {
+ for (const FreeList &list : small_lists)
+ list.sanitize(secrets);
+ large_trie.sanitize(secrets);
+}
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
diff --git a/libc/src/__support/freetrie.cpp b/libc/src/__support/freetrie.cpp
index 3dc93adf7ccf4..056a79b0b4f8a 100644
--- a/libc/src/__support/freetrie.cpp
+++ b/libc/src/__support/freetrie.cpp
@@ -10,10 +10,10 @@
namespace LIBC_NAMESPACE_DECL {
-void FreeTrie::remove(Node *node) {
+void FreeTrie::remove(Node *node, const FreeListSecrets &secrets) {
LIBC_ASSERT(!empty() && "cannot remove from empty trie");
FreeList list = node;
- list.pop(FreeListSecrets{});
+ list.pop(secrets);
Node *new_node = static_cast<Node *>(list.begin());
if (!new_node) {
// The freelist is empty. Replace the subtrie root with an arbitrary leaf.
@@ -61,4 +61,16 @@ void FreeTrie::replace_node(Node *node, Node *new_node) {
node->upper->parent = new_node;
}
+void FreeTrie::sanitize(const FreeListSecrets &secrets) const {
+ auto sanitize_trie_node = [&](auto &self, const Node *node) -> void {
+ if (!node)
+ return;
+ FreeList list = const_cast<Node *>(node);
+ list.sanitize(secrets);
+ self(self, node->lower);
+ self(self, node->upper);
+ };
+ sanitize_trie_node(sanitize_trie_node, root);
+}
+
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index ccc8c43836cfe..31c62543b04a5 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -101,15 +101,18 @@ class FreeTrie {
LIBC_INLINE bool empty() const { return !root; }
/// Push a block to the trie.
- void push(BlockRef block);
+ void push(BlockRef block, const FreeListSecrets &secrets);
/// Remove a node from this trie node's free list.
- void remove(Node *node);
+ void remove(Node *node, const FreeListSecrets &secrets);
/// @returns A smallest node that can allocate the given size; otherwise
/// nullptr.
Node *find_best_fit(size_t size);
+ /// Verify secret invariants for all free lists in the trie.
+ void sanitize(const FreeListSecrets &secrets) const;
+
private:
/// @returns Whether a node is the head of its containing freelist.
bool is_head(Node *node) const { return node->parent || node == root; }
@@ -122,7 +125,8 @@ class FreeTrie {
SizeRange range;
};
-LIBC_INLINE void FreeTrie::push(BlockRef block) {
+LIBC_INLINE void FreeTrie::push(BlockRef block,
+ const FreeListSecrets &secrets) {
LIBC_ASSERT(block.inner_size_free() >= sizeof(Node) &&
"block too small to accomodate free trie node");
size_t size = block.inner_size();
@@ -152,7 +156,7 @@ LIBC_INLINE void FreeTrie::push(BlockRef block) {
} else {
node->parent = nullptr;
}
- list.push(node, FreeListSecrets{});
+ list.push(node, secrets);
*cur = static_cast<Node *>(list.begin());
}
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index c4117c35049ec..57085275de7bb 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -358,3 +358,19 @@ TEST_FOR_EACH_ALLOCATOR(InvalidAlignedAllocAlignment, 2048) {
ptr = allocator.aligned_allocate(0, 8);
EXPECT_EQ(ptr, static_cast<void *>(nullptr));
}
+
+TEST_FOR_EACH_ALLOCATOR(SanitizeHeap, 2048) {
+ void *ptr1 = allocator.allocate(128);
+ void *ptr2 = allocator.allocate(256);
+ void *ptr3 = allocator.allocate(512);
+ EXPECT_NE(ptr1, static_cast<void *>(nullptr));
+ EXPECT_NE(ptr2, static_cast<void *>(nullptr));
+ EXPECT_NE(ptr3, static_cast<void *>(nullptr));
+
+ allocator.free(ptr2);
+ allocator.sanitize_heap();
+
+ allocator.free(ptr1);
+ allocator.free(ptr3);
+ allocator.sanitize_heap();
+}
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index bf3284b2faf64..954d62edfb3cf 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -17,10 +17,18 @@
#include "test/UnitTest/Test.h"
using LIBC_NAMESPACE::BlockRef;
+using LIBC_NAMESPACE::FreeListSecrets;
using LIBC_NAMESPACE::FreeTrie;
using LIBC_NAMESPACE::cpp::byte;
using LIBC_NAMESPACE::cpp::optional;
+#if LIBC_COPT_HARDEN_FREELIST
+#define TEST_SECRETS FreeListSecrets{0x1234, 0x5678, 0x9abc}
+#else
+#define TEST_SECRETS \
+ FreeListSecrets {}
+#endif
+
TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
FreeTrie trie({0, 4096});
EXPECT_EQ(trie.find_best_fit(123), static_cast<FreeTrie::Node *>(nullptr));
@@ -29,7 +37,7 @@ TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
optional<BlockRef> maybeBlock = BlockRef::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
BlockRef block = *maybeBlock;
- trie.push(block);
+ trie.push(block, TEST_SECRETS);
FreeTrie::Node *root = trie.find_best_fit(0);
ASSERT_EQ(root->block().addr(), block.addr());
@@ -50,8 +58,8 @@ TEST(LlvmLibcFreeTrie, FindBestFitLower) {
BlockRef root = *maybeBlock;
FreeTrie trie({0, 4096});
- trie.push(root);
- trie.push(lower);
+ trie.push(root, TEST_SECRETS);
+ trie.push(lower, TEST_SECRETS);
EXPECT_EQ(trie.find_best_fit(0)->block().addr(), lower.addr());
}
@@ -66,8 +74,8 @@ TEST(LlvmLibcFreeTrie, FindBestFitUpper) {
BlockRef upper = *maybeBlock;
FreeTrie trie({0, 4096});
- trie.push(root);
- trie.push(upper);
+ trie.push(root, TEST_SECRETS);
+ trie.push(upper, TEST_SECRETS);
EXPECT_EQ(trie.find_best_fit(root.inner_size() + 1)->block().addr(),
upper.addr());
@@ -89,9 +97,9 @@ TEST(LlvmLibcFreeTrie, FindBestFitLowerAndUpper) {
BlockRef upper = *maybeBlock;
FreeTrie trie({0, 4096});
- trie.push(root);
- trie.push(lower);
- trie.push(upper);
+ trie.push(root, TEST_SECRETS);
+ trie.push(lower, TEST_SECRETS);
+ trie.push(upper, TEST_SECRETS);
// The lower subtrie is examined first.
EXPECT_EQ(trie.find_best_fit(0)->block().addr(), lower.addr());
@@ -115,20 +123,20 @@ TEST(LlvmLibcFreeTrie, Remove) {
// Removing the root empties the trie.
FreeTrie trie({0, 4096});
- trie.push(large);
+ trie.push(large, TEST_SECRETS);
FreeTrie::Node *large_node = trie.find_best_fit(0);
ASSERT_EQ(large_node->block().addr(), large.addr());
- trie.remove(large_node);
+ trie.remove(large_node, TEST_SECRETS);
ASSERT_TRUE(trie.empty());
// Removing the head of a trie list preserves the trie structure.
- trie.push(small1);
- trie.push(small2);
- trie.push(large);
- trie.remove(trie.find_best_fit(small1.inner_size()));
+ trie.push(small1, TEST_SECRETS);
+ trie.push(small2, TEST_SECRETS);
+ trie.push(large, TEST_SECRETS);
+ trie.remove(trie.find_best_fit(small1.inner_size()), TEST_SECRETS);
EXPECT_EQ(trie.find_best_fit(large.inner_size())->block().addr(),
large.addr());
- trie.remove(trie.find_best_fit(small1.inner_size()));
+ trie.remove(trie.find_best_fit(small1.inner_size()), TEST_SECRETS);
EXPECT_EQ(trie.find_best_fit(large.inner_size())->block().addr(),
large.addr());
}
More information about the libc-commits
mailing list