[libc-commits] [libc] [llvm] freelist clz optimization (PR #205625)
via libc-commits
libc-commits at lists.llvm.org
Wed Jun 24 13:02:17 PDT 2026
github-actions[bot] wrote:
<!--LLVM CODE FORMAT COMMENT: {clang-format}-->
:warning: C/C++ code formatter, clang-format found issues in your code. :warning:
<details>
<summary>
You can test this locally with the following command:
</summary>
``````````bash
git-clang-format --diff origin/main HEAD --extensions h,cpp -- libc/src/__support/freelist.h libc/src/__support/freelist_heap.h libc/src/__support/freestore.h libc/src/__support/freetrie.h libc/test/src/__support/freelist_heap_test.cpp libc/test/src/__support/freelist_test.cpp libc/test/src/__support/freestore_test.cpp libc/test/src/__support/freetrie_test.cpp --diff_from_common_commit
``````````
:warning:
The reproduction instructions above might return results for more than one PR
in a stack if you are using a stacked PR workflow. You can limit the results by
changing `origin/main` to the base branch/commit you want to compare against.
:warning:
</details>
<details>
<summary>
View the diff from clang-format here.
</summary>
``````````diff
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index 46b1893f0..9ba3bac04 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -24,11 +24,11 @@
#endif
#if LIBC_COPT_HARDEN_FREELIST
-#define LIBC_HARDENING_ASSERT(cond) \
- do { \
- if (LIBC_UNLIKELY(!(cond))) { \
- __builtin_trap(); \
- } \
+#define LIBC_HARDENING_ASSERT(cond) \
+ do { \
+ if (LIBC_UNLIKELY(!(cond))) { \
+ __builtin_trap(); \
+ } \
} while (0)
#else
#define LIBC_HARDENING_ASSERT(cond) LIBC_ASSERT(cond)
@@ -43,41 +43,39 @@ struct FreeListSecrets {
uintptr_t k2;
#endif
- template <typename T>
- LIBC_INLINE T* decrypt_next(T* next_val) const {
+ template <typename T> LIBC_INLINE T *decrypt_next(T *next_val) const {
#if LIBC_COPT_HARDEN_FREELIST
- return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(next_val) ^ k0);
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(next_val) ^ k0);
#else
return next_val;
#endif
}
template <typename T>
- LIBC_INLINE T* decrypt_prev([[maybe_unused]] const void* node,
- T* prev_val) const {
+ LIBC_INLINE T *decrypt_prev([[maybe_unused]] const void *node,
+ T *prev_val) const {
#if LIBC_COPT_HARDEN_FREELIST
- return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(prev_val) ^ k1 ^
- reinterpret_cast<uintptr_t>(node) ^ k2);
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(prev_val) ^ k1 ^
+ reinterpret_cast<uintptr_t>(node) ^ k2);
#else
return prev_val;
#endif
}
- template <typename T>
- LIBC_INLINE T* encrypt_next(T* next_val) const {
+ template <typename T> LIBC_INLINE T *encrypt_next(T *next_val) const {
#if LIBC_COPT_HARDEN_FREELIST
- return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(next_val) ^ k0);
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(next_val) ^ k0);
#else
return next_val;
#endif
}
template <typename T>
- LIBC_INLINE T* encrypt_prev([[maybe_unused]] const void* node,
- T* prev_val) const {
+ LIBC_INLINE T *encrypt_prev([[maybe_unused]] const void *node,
+ T *prev_val) const {
#if LIBC_COPT_HARDEN_FREELIST
- return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(prev_val) ^ k1 ^
- reinterpret_cast<uintptr_t>(node) ^ k2);
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(prev_val) ^ k1 ^
+ reinterpret_cast<uintptr_t>(node) ^ k2);
#else
return prev_val;
#endif
@@ -131,19 +129,19 @@ public:
/// @returns The first block in the list.
LIBC_INLINE BlockRef front() { return begin_->block(); }
- LIBC_INLINE Node* next_node(const Node* node,
- const FreeListSecrets& secrets) const {
+ LIBC_INLINE Node *next_node(const Node *node,
+ const FreeListSecrets &secrets) const {
return node ? secrets.decrypt_next(node->next) : nullptr;
}
- LIBC_INLINE Node* prev_node(const Node* node,
- const FreeListSecrets& secrets) const {
+ LIBC_INLINE Node *prev_node(const Node *node,
+ const FreeListSecrets &secrets) const {
return node ? secrets.decrypt_prev(node, node->prev) : nullptr;
}
/// Push a block to the back of the list.
/// The block must be large enough to contain a node.
- LIBC_INLINE void push(BlockRef block, const FreeListSecrets& secrets) {
+ LIBC_INLINE void push(BlockRef block, const FreeListSecrets &secrets) {
LIBC_ASSERT(!block.used() &&
"only free blocks can be placed on free lists");
LIBC_ASSERT(block.inner_size_free() >= sizeof(Node) &&
@@ -153,9 +151,9 @@ public:
/// Push an already-constructed node to the back of the list.
/// This allows pushing derived node types with additional data.
- LIBC_INLINE void push(Node* node, const FreeListSecrets& secrets) {
+ LIBC_INLINE void push(Node *node, const FreeListSecrets &secrets) {
if (begin_) {
- Node* begin_prev = secrets.decrypt_prev(begin_, begin_->prev);
+ Node *begin_prev = secrets.decrypt_prev(begin_, begin_->prev);
LIBC_HARDENING_ASSERT(secrets.decrypt_next(begin_prev->next) == begin_ &&
"Corrupted free list links (push check)");
@@ -172,20 +170,20 @@ public:
}
/// Pop the first node from the list.
- LIBC_INLINE void pop(const FreeListSecrets& secrets) {
+ LIBC_INLINE void pop(const FreeListSecrets &secrets) {
remove(begin_, secrets);
}
/// Remove an arbitrary node from the list.
- LIBC_INLINE void remove(Node* node, const FreeListSecrets& secrets) {
+ LIBC_INLINE void remove(Node *node, const FreeListSecrets &secrets) {
LIBC_ASSERT(begin_ && "cannot remove from empty list");
- Node* node_next = secrets.decrypt_next(node->next);
+ Node *node_next = secrets.decrypt_next(node->next);
if (node == node_next) {
LIBC_ASSERT(node == begin_ &&
"a self-referential node must be the only element");
begin_ = nullptr;
} else {
- Node* node_prev = secrets.decrypt_prev(node, node->prev);
+ Node *node_prev = secrets.decrypt_prev(node, node->prev);
LIBC_HARDENING_ASSERT(
secrets.decrypt_next(node_prev->next) == node &&
@@ -196,7 +194,8 @@ public:
node_prev->next = secrets.encrypt_next(node_next);
node_next->prev = secrets.encrypt_prev(node_next, node_prev);
- if (begin_ == node) begin_ = node_next;
+ if (begin_ == node)
+ begin_ = node_next;
}
}
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index e82822825..f5e7700f0 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -50,32 +50,32 @@ LIBC_INLINE constexpr bool IsPow2(size_t x) { return x && (x & (x - 1)) == 0; }
class FreeListHeap {
public:
#if LIBC_COPT_HARDEN_FREELIST
- constexpr FreeListHeap(const FreeListSecrets& secrets =
- {LIBC_COPT_FREELIST_KEY0, LIBC_COPT_FREELIST_KEY1,
- LIBC_COPT_FREELIST_KEY2})
- : begin(&_end), end(&__llvm_libc_heap_limit), secrets(secrets) {}
-
- constexpr FreeListHeap(span<cpp::byte> region,
- const FreeListSecrets& secrets =
- {LIBC_COPT_FREELIST_KEY0, LIBC_COPT_FREELIST_KEY1,
- LIBC_COPT_FREELIST_KEY2})
- : begin(region.begin()), end(region.end()), secrets(secrets) {}
+ constexpr FreeListHeap(const FreeListSecrets &secrets =
+ {LIBC_COPT_FREELIST_KEY0, LIBC_COPT_FREELIST_KEY1,
+ LIBC_COPT_FREELIST_KEY2})
+ : begin(&_end), end(&__llvm_libc_heap_limit), secrets(secrets) {}
+
+ constexpr FreeListHeap(span<cpp::byte> region,
+ const FreeListSecrets &secrets =
+ {LIBC_COPT_FREELIST_KEY0, LIBC_COPT_FREELIST_KEY1,
+ LIBC_COPT_FREELIST_KEY2})
+ : begin(region.begin()), end(region.end()), secrets(secrets) {}
#else
- constexpr FreeListHeap() : begin(&_end), end(&__llvm_libc_heap_limit) {}
+ constexpr FreeListHeap() : begin(&_end), end(&__llvm_libc_heap_limit) {}
- constexpr FreeListHeap(span<cpp::byte> region)
- : begin(region.begin()), end(region.end()) {}
+ constexpr FreeListHeap(span<cpp::byte> region)
+ : begin(region.begin()), end(region.end()) {}
#endif
- void* allocate(size_t size);
- void* aligned_allocate(size_t alignment, size_t size);
- // NOTE: All pointers passed to free must come from one of the other
- // allocation functions: `allocate`, `aligned_allocate`, `realloc`, `calloc`.
- void free(void* ptr);
- void* realloc(void* ptr, size_t size);
- void* calloc(size_t num, size_t size);
+ void *allocate(size_t size);
+ void *aligned_allocate(size_t alignment, size_t size);
+ // NOTE: All pointers passed to free must come from one of the other
+ // allocation functions: `allocate`, `aligned_allocate`, `realloc`, `calloc`.
+ void free(void *ptr);
+ void *realloc(void *ptr, size_t size);
+ void *calloc(size_t num, size_t size);
- cpp::span<cpp::byte> region() const { return {begin, end}; }
+ cpp::span<cpp::byte> region() const { return {begin, end}; }
private:
void init();
@@ -102,13 +102,13 @@ private:
template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
public:
#if LIBC_COPT_HARDEN_FREELIST
- constexpr FreeListHeapBuffer(
- const FreeListSecrets& secrets = {LIBC_COPT_FREELIST_KEY0,
- LIBC_COPT_FREELIST_KEY1,
- LIBC_COPT_FREELIST_KEY2})
- : FreeListHeap{buffer, secrets}, buffer{} {}
+ constexpr FreeListHeapBuffer(
+ const FreeListSecrets &secrets = {LIBC_COPT_FREELIST_KEY0,
+ LIBC_COPT_FREELIST_KEY1,
+ LIBC_COPT_FREELIST_KEY2})
+ : FreeListHeap{buffer, secrets}, buffer{} {}
#else
- constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
+ constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
#endif
private:
@@ -118,7 +118,7 @@ private:
#if LIBC_COPT_HARDEN_FREELIST
#define HEAP_SECRETS secrets
#else
-#define HEAP_SECRETS \
+#define HEAP_SECRETS \
FreeListSecrets {}
#endif
@@ -130,8 +130,8 @@ private:
is_initialized = true;
}
-[[gnu::noinline]] LIBC_INLINE void* FreeListHeap::allocate_impl(
- size_t alignment, size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *
+FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (size == 0)
return nullptr;
@@ -147,19 +147,21 @@ private:
return nullptr;
auto block_info = BlockRef::allocate(block, alignment, size);
- if (block_info.next) free_store.insert(block_info.next, HEAP_SECRETS);
- if (block_info.prev) free_store.insert(block_info.prev, HEAP_SECRETS);
+ if (block_info.next)
+ free_store.insert(block_info.next, HEAP_SECRETS);
+ if (block_info.prev)
+ free_store.insert(block_info.prev, HEAP_SECRETS);
block_info.block.mark_used();
return block_info.block.usable_space();
}
-[[gnu::noinline]] LIBC_INLINE void* FreeListHeap::allocate(size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *FreeListHeap::allocate(size_t size) {
return allocate_impl(BlockRef::MIN_ALIGN, size);
}
-[[gnu::noinline]] LIBC_INLINE void* FreeListHeap::aligned_allocate(
- size_t alignment, size_t size) {
+[[gnu::noinline]] LIBC_INLINE void *
+FreeListHeap::aligned_allocate(size_t alignment, size_t size) {
// The alignment must be an integral power of two.
if (!IsPow2(alignment))
return nullptr;
@@ -174,7 +176,7 @@ private:
return allocate_impl(alignment, size);
}
-[[gnu::noinline]] LIBC_INLINE void FreeListHeap::free(void* ptr) {
+[[gnu::noinline]] LIBC_INLINE void FreeListHeap::free(void *ptr) {
if (ptr == nullptr)
return;
@@ -235,7 +237,7 @@ private:
// Follows constract of the C standard realloc() function
// If ptr is free'd, will return nullptr.
-[[gnu::noinline]] LIBC_INLINE void* FreeListHeap::realloc(void* ptr,
+[[gnu::noinline]] LIBC_INLINE void *FreeListHeap::realloc(void *ptr,
size_t size) {
if (size == 0) {
free(ptr);
@@ -271,7 +273,7 @@ private:
return new_ptr;
}
-[[gnu::noinline]] LIBC_INLINE void* FreeListHeap::calloc(size_t num,
+[[gnu::noinline]] LIBC_INLINE void *FreeListHeap::calloc(size_t num,
size_t size) {
size_t bytes;
if (__builtin_mul_overflow(num, size, &bytes))
@@ -283,7 +285,7 @@ private:
}
LIBC_INLINE_VAR LIBC_CONSTINIT FreeListHeap freelist_heap_symbols;
-LIBC_INLINE_VAR FreeListHeap* freelist_heap = &freelist_heap_symbols;
+LIBC_INLINE_VAR FreeListHeap *freelist_heap = &freelist_heap_symbols;
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index bc2981d22..f9dfc64b3 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -98,7 +98,7 @@ public:
void push(BlockRef block);
/// Remove a node from this trie node's free list.
- LIBC_INLINE void remove(Node* node);
+ LIBC_INLINE void remove(Node *node);
/// @returns A smallest node that can allocate the given size; otherwise
/// nullptr.
@@ -119,7 +119,7 @@ private:
/// Replaces references to one node with another (or nullptr) in all adjacent
/// parent and child nodes.
- LIBC_INLINE void replace_node(Node* node, Node* new_node);
+ LIBC_INLINE void replace_node(Node *node, Node *new_node);
Node *&root;
SizeRange range;
@@ -281,16 +281,16 @@ LIBC_INLINE FreeTrie::Node *FreeTrie::pop_min() {
return min_node;
}
-LIBC_INLINE void FreeTrie::remove(Node* node) {
+LIBC_INLINE void FreeTrie::remove(Node *node) {
LIBC_ASSERT(!empty() && "cannot remove from empty trie");
FreeList list = node;
list.pop(FreeListSecrets{});
- Node* new_node = static_cast<Node*>(list.begin());
+ Node *new_node = static_cast<Node *>(list.begin());
if (!new_node) {
// The freelist is empty. Replace the subtrie root with an arbitrary leaf.
// This is legal because there is no relationship between the size of the
// root and its children.
- Node* leaf = node;
+ Node *leaf = node;
while (leaf->lower || leaf->upper)
leaf = leaf->lower ? leaf->lower : leaf->upper;
if (leaf == node) {
@@ -303,7 +303,8 @@ LIBC_INLINE void FreeTrie::remove(Node* node) {
new_node = leaf;
}
- if (!is_head(node)) return;
+ if (!is_head(node))
+ return;
// Copy the trie links to the new head.
new_node->lower = node->lower;
@@ -312,11 +313,11 @@ LIBC_INLINE void FreeTrie::remove(Node* node) {
replace_node(node, new_node);
}
-LIBC_INLINE void FreeTrie::replace_node(Node* node, Node* new_node) {
+LIBC_INLINE void FreeTrie::replace_node(Node *node, Node *new_node) {
LIBC_ASSERT(is_head(node) && "only head nodes contain trie links");
if (node->parent) {
- Node*& parent_child =
+ Node *&parent_child =
node->parent->lower == node ? node->parent->lower : node->parent->upper;
LIBC_ASSERT(parent_child == node &&
"no reference to child node found in parent");
@@ -325,8 +326,10 @@ LIBC_INLINE void FreeTrie::replace_node(Node* node, Node* new_node) {
LIBC_ASSERT(root == node && "non-root node had no parent");
root = new_node;
}
- if (node->lower) node->lower->parent = new_node;
- if (node->upper) node->upper->parent = new_node;
+ if (node->lower)
+ node->lower->parent = new_node;
+ if (node->upper)
+ node->upper->parent = new_node;
}
} // namespace LIBC_NAMESPACE_DECL
``````````
</details>
https://github.com/llvm/llvm-project/pull/205625
More information about the libc-commits
mailing list