[libc-commits] [libc] [libc][malloc] add a flat TLSF allocator for baremetal environment (PR #200075)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Thu May 28 10:10:44 PDT 2026
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/200075
>From 63f249bacebd9c3dce317a81ebdfb0e0d93adcfe Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 27 May 2026 15:50:44 -0700
Subject: [PATCH 1/8] [libc][malloc] add a flat TLSF allocator for baremetal
environment
---
.../modules/LLVMLibCCompileOptionRules.cmake | 8 +
libc/config/config.json | 6 +
libc/src/__support/CMakeLists.txt | 18 +
libc/src/__support/flat_tlsf/CMakeLists.txt | 117 ++++
libc/src/__support/flat_tlsf/binning.h | 161 ++++++
libc/src/__support/flat_tlsf/bit_utils.h | 88 +++
libc/src/__support/flat_tlsf/bitfield.h | 73 +++
libc/src/__support/flat_tlsf/chunk.h | 91 ++++
libc/src/__support/flat_tlsf/common.h | 35 ++
libc/src/__support/flat_tlsf/global.cpp | 24 +
libc/src/__support/flat_tlsf/global.h | 117 ++++
libc/src/__support/flat_tlsf/heap.h | 504 ++++++++++++++++++
libc/src/__support/flat_tlsf/node.h | 47 ++
libc/src/__support/flat_tlsf/tag.h | 50 ++
libc/src/__support/heap.cpp | 29 +
libc/src/__support/heap.h | 38 ++
libc/src/stdlib/baremetal/CMakeLists.txt | 10 +-
libc/src/stdlib/baremetal/aligned_alloc.cpp | 4 +-
libc/src/stdlib/baremetal/calloc.cpp | 4 +-
libc/src/stdlib/baremetal/free.cpp | 4 +-
libc/src/stdlib/baremetal/malloc.cpp | 4 +-
libc/src/stdlib/baremetal/realloc.cpp | 4 +-
libc/test/src/__support/CMakeLists.txt | 1 +
.../src/__support/flat_tlsf/CMakeLists.txt | 97 ++++
.../src/__support/flat_tlsf/binning_test.cpp | 150 ++++++
.../src/__support/flat_tlsf/bitfield_test.cpp | 195 +++++++
.../src/__support/flat_tlsf/chunk_test.cpp | 77 +++
.../src/__support/flat_tlsf/global_test.cpp | 286 ++++++++++
.../src/__support/flat_tlsf/heap_test.cpp | 219 ++++++++
.../src/__support/flat_tlsf/node_test.cpp | 88 +++
.../test/src/__support/flat_tlsf/tag_test.cpp | 58 ++
31 files changed, 2592 insertions(+), 15 deletions(-)
create mode 100644 libc/src/__support/flat_tlsf/CMakeLists.txt
create mode 100644 libc/src/__support/flat_tlsf/binning.h
create mode 100644 libc/src/__support/flat_tlsf/bit_utils.h
create mode 100644 libc/src/__support/flat_tlsf/bitfield.h
create mode 100644 libc/src/__support/flat_tlsf/chunk.h
create mode 100644 libc/src/__support/flat_tlsf/common.h
create mode 100644 libc/src/__support/flat_tlsf/global.cpp
create mode 100644 libc/src/__support/flat_tlsf/global.h
create mode 100644 libc/src/__support/flat_tlsf/heap.h
create mode 100644 libc/src/__support/flat_tlsf/node.h
create mode 100644 libc/src/__support/flat_tlsf/tag.h
create mode 100644 libc/src/__support/heap.cpp
create mode 100644 libc/src/__support/heap.h
create mode 100644 libc/test/src/__support/flat_tlsf/CMakeLists.txt
create mode 100644 libc/test/src/__support/flat_tlsf/binning_test.cpp
create mode 100644 libc/test/src/__support/flat_tlsf/bitfield_test.cpp
create mode 100644 libc/test/src/__support/flat_tlsf/chunk_test.cpp
create mode 100644 libc/test/src/__support/flat_tlsf/global_test.cpp
create mode 100644 libc/test/src/__support/flat_tlsf/heap_test.cpp
create mode 100644 libc/test/src/__support/flat_tlsf/node_test.cpp
create mode 100644 libc/test/src/__support/flat_tlsf/tag_test.cpp
diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index c65fc4db605c9..9ea0e820636ec 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -177,6 +177,14 @@ function(_get_compile_options_from_config output_var)
libc_add_definition(config_options "LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT=${LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT}")
endif()
+ if(LIBC_CONF_HEAP_TYPE)
+ if(LIBC_CONF_HEAP_TYPE STREQUAL "LIBC_HEAP_TYPE_FLAT_TLSF")
+ libc_add_definition(config_options "LIBC_HEAP_TYPE_FLAT_TLSF")
+ else()
+ libc_add_definition(config_options "LIBC_HEAP_TYPE_FREELIST")
+ endif()
+ endif()
+
if(LIBC_CONF_MATH_USE_SYSTEM_FENV)
libc_add_definition(config_options "LIBC_MATH_USE_SYSTEM_FENV")
endif()
diff --git a/libc/config/config.json b/libc/config/config.json
index 27d9f08ed31e0..ebeba5bd00706 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -196,5 +196,11 @@
"value": false,
"doc": "Use the system assert macro for LIBC_ASSERT."
}
+ },
+ "heap": {
+ "LIBC_CONF_HEAP_TYPE": {
+ "value": "LIBC_HEAP_TYPE_FREELIST",
+ "doc": "Selects the implementation for the global heap: 'LIBC_HEAP_TYPE_FREELIST' or 'LIBC_HEAP_TYPE_FLAT_TLSF'."
+ }
}
}
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 1f77fc02380df..2605ac36ed39a 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -82,6 +82,23 @@ add_object_library(
libc.src.string.memory_utils.inline_memset
)
+if(LIBC_CONF_HEAP_TYPE STREQUAL "LIBC_HEAP_TYPE_FLAT_TLSF")
+ set(heap_impl_dependency libc.src.__support.flat_tlsf.global)
+else()
+ set(heap_impl_dependency .freelist_heap)
+endif()
+
+add_object_library(
+ heap
+ HDRS
+ heap.h
+ SRCS
+ heap.cpp
+ DEPENDS
+ ${heap_impl_dependency}
+ libc.src.__support.macros.config
+)
+
add_header_library(
blockstore
HDRS
@@ -464,6 +481,7 @@ add_subdirectory(File)
add_subdirectory(HashTable)
add_subdirectory(fixed_point)
+add_subdirectory(flat_tlsf)
add_subdirectory(time)
diff --git a/libc/src/__support/flat_tlsf/CMakeLists.txt b/libc/src/__support/flat_tlsf/CMakeLists.txt
new file mode 100644
index 0000000000000..b57e6ce8aec3b
--- /dev/null
+++ b/libc/src/__support/flat_tlsf/CMakeLists.txt
@@ -0,0 +1,117 @@
+add_header_library(
+ common
+ HDRS
+ common.h
+ DEPENDS
+ libc.src.__support.macros.config
+)
+
+add_header_library(
+ bit_utils
+ HDRS
+ bit_utils.h
+ DEPENDS
+ libc.hdr.stdint_proxy
+ libc.src.__support.CPP.bit
+ libc.src.__support.CPP.limits
+ .common
+ libc.src.__support.macros.attributes
+ libc.src.__support.macros.config
+ libc.src.__support.math_extras
+)
+
+add_header_library(
+ bitfield
+ HDRS
+ bitfield.h
+ DEPENDS
+ libc.hdr.stdint_proxy
+ libc.src.__support.CPP.array
+ .bit_utils
+ libc.src.__support.macros.attributes
+ libc.src.__support.macros.config
+)
+
+add_header_library(
+ binning
+ HDRS
+ binning.h
+ DEPENDS
+ libc.hdr.stdint_proxy
+ .common
+ .bitfield
+ .bit_utils
+ libc.src.__support.macros.attributes
+ libc.src.__support.macros.config
+)
+
+add_header_library(
+ tag
+ HDRS
+ tag.h
+ DEPENDS
+ .common
+ libc.src.__support.macros.attributes
+ libc.src.__support.macros.config
+)
+
+add_header_library(
+ node
+ HDRS
+ node.h
+ DEPENDS
+ libc.src.__support.macros.attributes
+ libc.src.__support.macros.config
+)
+
+add_header_library(
+ chunk
+ HDRS
+ chunk.h
+ DEPENDS
+ .common
+ .node
+ .bit_utils
+ libc.src.__support.macros.attributes
+ libc.src.__support.macros.config
+ libc.src.string.memory_utils.inline_memcpy
+)
+
+add_header_library(
+ heap
+ HDRS
+ heap.h
+ DEPENDS
+ .common
+ .node
+ .bit_utils
+ .bitfield
+ .binning
+ .chunk
+ .tag
+ libc.hdr.types.size_t
+ libc.src.__support.CPP.optional
+ libc.src.__support.CPP.limits
+ libc.src.__support.CPP.algorithm
+ libc.src.string.memory_utils.inline_memcpy
+ libc.src.__support.libc_assert
+ libc.src.__support.macros.attributes
+ libc.src.__support.macros.config
+)
+
+add_object_library(
+ global
+ HDRS
+ global.h
+ SRCS
+ global.cpp
+ DEPENDS
+ .common
+ .heap
+ libc.src.__support.CPP.cstddef
+ libc.src.__support.CPP.span
+ libc.src.__support.math_extras
+ libc.src.string.memory_utils.inline_memset
+ libc.src.__support.macros.attributes
+ libc.src.__support.macros.config
+)
diff --git a/libc/src/__support/flat_tlsf/binning.h b/libc/src/__support/flat_tlsf/binning.h
new file mode 100644
index 0000000000000..6745b19c201ac
--- /dev/null
+++ b/libc/src/__support/flat_tlsf/binning.h
@@ -0,0 +1,161 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide Binning class for the flat_tlsf allocator.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_BINNING_H
+#define LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_BINNING_H
+
+#include "hdr/stdint_proxy.h"
+#include "src/__support/flat_tlsf/bit_utils.h"
+#include "src/__support/flat_tlsf/bitfield.h"
+#include "src/__support/flat_tlsf/common.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace flat_tlsf {
+
+struct Binning {
+ static_assert(sizeof(void *) == 4 || sizeof(void *) == 8,
+ "Only 32-bit and 64-bit architectures are currently supported");
+
+ static constexpr size_t BIN_COUNT = BitField::BITS - 1;
+
+ /// A fast binning algorithm with relatively even coverage and configurable
+ /// behavior.
+ ///
+ /// This is the default binning algorithm used due to having a good spread of
+ /// bin intervals, being able to take advantage of many or few buckets well,
+ /// and being very fast (only a handful of instructions with one branch).
+ ///
+ /// # Behavior by size
+ /// - `0..=(CHUNK_UNIT*LIN_DIVS*LIN_EXT_MULTI)` : Bins sizes into
+ /// one-bin-per-chunk-size
+ /// - `(CHUNK_UNIT*LIN_DIVS*LIN_EXT_MULTI)..` : Binds sizes by
+ /// linearly-subdivided exponential levels.
+ ///
+ /// # Parameters
+ /// - `LIN_DIVS`: the number of linear regions per power of two in the
+ /// exponential region.
+ /// - The higher this is, the more buckets are needed but the binning is
+ /// more fine-grained.
+ /// - Must be a power of two.
+ /// - Typically 2 (few bins, subpar granularity), 4, or 8 (lots of bins,
+ /// good granularity).
+ /// - This is the parameter you want to figure out first for a given
+ /// number of bins.
+ ///
+ /// - `LIN_EXT_MULTI`: the linear region extent multiplier.
+ /// - Scales the extent of the linear region.
+ /// - Must be a power of two.
+ /// - Set this to 1 by default.
+ /// - If there are too many bins being used on excessively-high size
+ /// regions, this is useful for spending those bins on more buckets for
+ /// small sizes instead.
+ ///
+ /// # Deciding on the parameters
+ /// `LIN_DIVS` has a much larger effect so tinker with that first while
+ /// keeping `LIN_EXT_MULTI` low, and then increase `LIN_EXT_MULTI` if there is
+ /// useless range at the top, given the number of bins you have.
+ ///
+ /// Having a range up to around 128MiB~2GiB is enough for most applications.
+ /// But keep in mind the largest bucket size you'll ever make use of is the
+ /// largest contiguous span of memory.
+ ///
+ /// The main effects on the allocator will be the heap efficiency and the
+ /// performance.
+ template <size_t LIN_DIVS, size_t LIN_EXT_MULTI>
+ LIBC_INLINE static constexpr size_t
+ linear_extend_then_linearly_divided_expotential_binning(size_t size) {
+ static_assert(bit_utils::is_power_of_2(LIN_DIVS),
+ "LIN_DIVS must be a power of two");
+ static_assert(bit_utils::is_power_of_2(LIN_EXT_MULTI),
+ "LIN_EXT_MULTI must be a power of two");
+
+ size_t exponential_region = CHUNK_UNIT * LIN_DIVS * LIN_EXT_MULTI;
+
+ // If the size is small enough, just divide by the chunk size.
+ // This is a fast short-circuit that handles the case where the size is
+ // smaller than `CHUNK_UNIT` and doesn't waste extra bins due to exponential
+ // subdivisions being smaller than `CHUNK_UNIT` here.
+ if (size <= exponential_region)
+ return size >> bit_utils::ilog2(CHUNK_UNIT);
+
+ // Let's say `exponential_region` is 256, the chunk unit is 32, LIN_DIVS is
+ // 4
+ //
+ // Exponential level 0: 256 ; (512 - 256)/LIN_DIVS = 256/LIN_DIVS = 64
+ // Subdiv 0: 256 ; bin 0 + LIN_DIVS * LIN_EXT_MULTI
+ // Subdiv 1: 256 + 64 ; bin 1 + LIN_DIVS * LIN_EXT_MULTI
+ // Subdiv 2: 256 + 128 ; bin 2 + LIN_DIVS * LIN_EXT_MULTI
+ // Subdiv 3: 256 + 196 ; bin 3 + LIN_DIVS * LIN_EXT_MULTI
+ // Exponential level 1: 512 ; 512/LIN_DIVS = 128
+ // Subdiv 0: 512 ; bin 4 + LIN_DIVS * LIN_EXT_MULTI
+ // Subdiv 1: 512 + 128 ; bin 5 + LIN_DIVS * LIN_EXT_MULTI
+ // Subdiv 2: 512 + 256 ; bin 6 + LIN_DIVS * LIN_EXT_MULTI
+ // Subdiv 3: 512 + 384 ; bin 7 + LIN_DIVS * LIN_EXT_MULTI
+ // Exponential level 2: 1024 ; 1024/LIN_DIVS = 256
+ // etc...
+ //
+ // Any size here is essentially broken up as follows:
+ //
+ // 00000000_1_01_010101010
+ // ^^^^^^^^^ dead bits; all of this is ignored, effectively
+ // rounding down these bits away
+ // ^^ linear division bits; LIN_DIVS.ilog2() bits long after the
+ // first set bit; tells us which linear subdivision we're in
+ // ^ first set bit; dictates size.ilog2(); tells us which
+ // "exponential level" this size is
+
+ size_t size_ilog2 = bit_utils::ilog2(size);
+
+ // Shift out the dead bits. This leaves the linear subdivision plus LIN_DIVS
+ // (due to the always-set bit at the top)
+ size_t linear_subdivision_plus_lin_divs =
+ size >> (size_ilog2 - bit_utils::ilog2(LIN_DIVS));
+
+ // Extract the exponential level above the `exponential_region` limit
+ // add LIN_EXT_MULTI here along with the other constants, it will get
+ // multiplied by LIN_DIVS next which gives us the exponential bins offset
+ // subtract 1 along with the other constants, this is important later
+ size_t unshifted_exponential_minus_one =
+ size_ilog2 - bit_utils::ilog2(exponential_region) + LIN_EXT_MULTI - 1;
+
+ // Multiply the exponential level by LIN_DIVS to shift it above the linear
+ // division bits Multiply the LIN_EXT_MULTI by LIN_DIVS to add the offset
+ // due to the linearly-spaced buckets Multiply (-1) to get (-LIN_DIVS)
+ size_t exponential_plus_offset_minus_lin_divs =
+ unshifted_exponential_minus_one << bit_utils::ilog2(LIN_DIVS);
+
+ // This LIN_DIVS cancel out, yielding the expected exponential-region bin
+ return exponential_plus_offset_minus_lin_divs +
+ linear_subdivision_plus_lin_divs;
+ }
+
+ LIBC_INLINE constexpr static uint32_t size_to_bin(size_t size) {
+ if constexpr (sizeof(void *) == 8)
+ return static_cast<uint32_t>(
+ linear_extend_then_linearly_divided_expotential_binning<8, 4>(size));
+ else
+ return static_cast<uint32_t>(
+ linear_extend_then_linearly_divided_expotential_binning<4, 4>(size));
+ }
+
+ LIBC_INLINE constexpr static uint32_t size_to_bin_ceil(size_t size) {
+ return size_to_bin(size - 1) + 1;
+ }
+};
+
+} // namespace flat_tlsf
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_BINNING_H
diff --git a/libc/src/__support/flat_tlsf/bit_utils.h b/libc/src/__support/flat_tlsf/bit_utils.h
new file mode 100644
index 0000000000000..cf52c8b34b2e5
--- /dev/null
+++ b/libc/src/__support/flat_tlsf/bit_utils.h
@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide bit utilities for the flat_tlsf allocator.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_BIT_UTILS_H
+#define LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_BIT_UTILS_H
+
+#include "hdr/stdint_proxy.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/flat_tlsf/common.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/math_extras.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace flat_tlsf {
+
+// We could use cpp::byte, but that definition currently have strict aliasing
+// violation, hence we fallback to unsigned char directly.
+namespace bit_utils {
+
+LIBC_INLINE constexpr size_t ilog2(size_t n) {
+ return cpp::numeric_limits<size_t>::digits - 1 -
+ static_cast<size_t>(cpp::countl_zero(n));
+}
+
+LIBC_INLINE constexpr bool is_power_of_2(size_t n) {
+ return cpp::has_single_bit(n);
+}
+
+LIBC_INLINE constexpr uint32_t bit_scan_after(size_t w, uint32_t start_index) {
+ size_t lower_bits_cleared = (w >> start_index) << start_index;
+ return static_cast<uint32_t>(cpp::countr_zero(lower_bits_cleared));
+}
+
+LIBC_INLINE constexpr void set_bit(size_t &w, uint32_t index) {
+ w |= size_t{1} << index;
+}
+
+LIBC_INLINE constexpr void clear_bit(size_t &w, uint32_t index) {
+ w &= ~(size_t{1} << index);
+}
+
+LIBC_INLINE constexpr bool read_bit(size_t w, uint32_t index) {
+ return w & (size_t{1} << index);
+}
+
+LIBC_INLINE bool is_aligned_to(Byte *ptr, size_t align) {
+ return (cpp::bit_cast<uintptr_t>(ptr) & (align - 1)) == 0;
+}
+
+LIBC_INLINE Byte *align_down_by(Byte *ptr, size_t align) {
+ uintptr_t addr = cpp::bit_cast<uintptr_t>(ptr);
+ return cpp::bit_cast<Byte *>(addr & ~(align - 1));
+}
+
+LIBC_INLINE Byte *align_up_by_mask(Byte *ptr, size_t align_mask) {
+ uintptr_t addr = cpp::bit_cast<uintptr_t>(ptr);
+ return cpp::bit_cast<Byte *>((addr + align_mask) & ~align_mask);
+}
+
+LIBC_INLINE Byte *align_up_by(Byte *ptr, size_t align) {
+ return align_up_by_mask(ptr, align - 1);
+}
+
+LIBC_INLINE Byte *saturating_ptr_add(Byte *ptr, size_t bytes) {
+ uintptr_t addr = cpp::bit_cast<uintptr_t>(ptr);
+ uintptr_t result;
+ if (add_overflow(addr, bytes, result))
+ return cpp::bit_cast<Byte *>(cpp::numeric_limits<uintptr_t>::max());
+ return cpp::bit_cast<Byte *>(result);
+}
+
+} // namespace bit_utils
+} // namespace flat_tlsf
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_BIT_UTILS_H
diff --git a/libc/src/__support/flat_tlsf/bitfield.h b/libc/src/__support/flat_tlsf/bitfield.h
new file mode 100644
index 0000000000000..93b4dc895492c
--- /dev/null
+++ b/libc/src/__support/flat_tlsf/bitfield.h
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide BitField class for the flat_tlsf allocator.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_BITFIELD_H
+#define LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_BITFIELD_H
+
+#include "hdr/stdint_proxy.h"
+#include "src/__support/CPP/array.h"
+#include "src/__support/flat_tlsf/bit_utils.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace flat_tlsf {
+
+struct alignas(16) BitField {
+ static constexpr size_t BITS_PER_ELEMENT = CHAR_WIDTH * sizeof(size_t);
+ static constexpr size_t NUMBER_OF_ELEMENTS = 3;
+ static constexpr size_t BITS = BITS_PER_ELEMENT * NUMBER_OF_ELEMENTS;
+
+ cpp::array<size_t, NUMBER_OF_ELEMENTS> storage;
+
+ LIBC_INLINE static constexpr BitField zeros() { return {}; }
+
+ LIBC_INLINE constexpr uint32_t bit_scan_after(uint32_t bit) const {
+ uint32_t array_index = bit / BITS_PER_ELEMENT;
+ uint32_t element_index = bit % BITS_PER_ELEMENT;
+ uint32_t bit_index =
+ bit_utils::bit_scan_after(storage[array_index], element_index);
+ if (bit_index < BITS_PER_ELEMENT)
+ return array_index * BITS_PER_ELEMENT + bit_index;
+ for (array_index = array_index + 1; array_index < NUMBER_OF_ELEMENTS;
+ ++array_index) {
+ bit_index = bit_utils::bit_scan_after(storage[array_index], 0);
+ if (bit_index < BITS_PER_ELEMENT)
+ return array_index * BITS_PER_ELEMENT + bit_index;
+ }
+ return BITS;
+ }
+
+ LIBC_INLINE constexpr void set_bit(uint32_t b) {
+ size_t array_index = b / BITS_PER_ELEMENT;
+ uint32_t element_index = static_cast<uint32_t>(b % BITS_PER_ELEMENT);
+ bit_utils::set_bit(storage[array_index], element_index);
+ }
+
+ LIBC_INLINE constexpr void clear_bit(uint32_t b) {
+ size_t array_index = b / BITS_PER_ELEMENT;
+ uint32_t element_index = static_cast<uint32_t>(b % BITS_PER_ELEMENT);
+ bit_utils::clear_bit(storage[array_index], element_index);
+ }
+
+ LIBC_INLINE constexpr bool read_bit(uint32_t b) const {
+ size_t array_index = b / BITS_PER_ELEMENT;
+ uint32_t element_index = static_cast<uint32_t>(b % BITS_PER_ELEMENT);
+ return bit_utils::read_bit(storage[array_index], element_index);
+ }
+};
+
+} // namespace flat_tlsf
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_BITFIELD_H
diff --git a/libc/src/__support/flat_tlsf/chunk.h b/libc/src/__support/flat_tlsf/chunk.h
new file mode 100644
index 0000000000000..62cfa56f62799
--- /dev/null
+++ b/libc/src/__support/flat_tlsf/chunk.h
@@ -0,0 +1,91 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide chunk utility functions for the flat_tlsf allocator.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_CHUNK_H
+#define LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_CHUNK_H
+
+#include "src/__support/flat_tlsf/bit_utils.h"
+#include "src/__support/flat_tlsf/common.h"
+#include "src/__support/flat_tlsf/node.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+#include "src/string/memory_utils/inline_memcpy.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace flat_tlsf {
+namespace chunk {
+
+LIBC_INLINE bool is_chunk_size(Byte *base, Byte *end) {
+ return end >= base + CHUNK_UNIT;
+}
+
+LIBC_INLINE constexpr size_t required_chunk_size(size_t size) {
+ size_t size_with_tag = size + 1;
+ size_t align_offset = (-size_with_tag) & (CHUNK_UNIT - 1);
+ return size_with_tag + align_offset;
+}
+
+LIBC_INLINE Byte *alloc_to_end(Byte *base, size_t size) {
+ return base + required_chunk_size(size);
+}
+
+LIBC_INLINE Node *gap_base_to_node(Byte *base) {
+ return reinterpret_cast<Node *>(base + GAP_NODE_OFFSET);
+}
+
+LIBC_INLINE uint32_t *gap_base_to_bin(Byte *base) {
+ return reinterpret_cast<uint32_t *>(base + GAP_BIN_OFFSET);
+}
+
+LIBC_INLINE size_t *gap_base_to_size(Byte *base) {
+ return reinterpret_cast<size_t *>(base + GAP_LOW_SIZE_OFFSET);
+}
+
+LIBC_INLINE size_t *gap_end_to_size_and_flag(Byte *end) {
+ return reinterpret_cast<size_t *>(end - GAP_HIGH_SIZE_OFFSET);
+}
+
+LIBC_INLINE Byte *gap_node_to_base(Node *node) {
+ return reinterpret_cast<Byte *>(node) - GAP_NODE_OFFSET;
+}
+
+LIBC_INLINE size_t *gap_node_to_size(Node *node) {
+ return reinterpret_cast<size_t *>(reinterpret_cast<Byte *>(node) -
+ GAP_NODE_OFFSET + GAP_LOW_SIZE_OFFSET);
+}
+
+LIBC_INLINE Byte *end_to_tag(Byte *end) { return end - sizeof(Byte); }
+
+LIBC_INLINE Byte *align_up(Byte *ptr) {
+ return bit_utils::align_up_by(ptr, CHUNK_UNIT);
+}
+
+LIBC_INLINE Byte *align_down(Byte *ptr) {
+ return bit_utils::align_down_by(ptr, CHUNK_UNIT);
+}
+
+template <class T> LIBC_INLINE T read_word(const void *ptr) {
+ T buffer;
+ inline_memcpy(&buffer, ptr, sizeof(T));
+ return buffer;
+}
+
+template <class T> LIBC_INLINE void write_word(void *ptr, T value) {
+ inline_memcpy(ptr, &value, sizeof(T));
+}
+
+} // namespace chunk
+} // namespace flat_tlsf
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_CHUNK_H
diff --git a/libc/src/__support/flat_tlsf/common.h b/libc/src/__support/flat_tlsf/common.h
new file mode 100644
index 0000000000000..02ffb924ea2a1
--- /dev/null
+++ b/libc/src/__support/flat_tlsf/common.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide common definitions and constants for the flat_tlsf allocator.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_COMMON_H
+#define LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_COMMON_H
+
+#include "src/__support/macros/config.h"
+#include <stddef.h>
+
+namespace LIBC_NAMESPACE_DECL {
+namespace flat_tlsf {
+
+using Byte = unsigned char;
+
+constexpr size_t CHUNK_UNIT = 4 * sizeof(size_t);
+
+constexpr size_t GAP_NODE_OFFSET = 0;
+constexpr size_t GAP_BIN_OFFSET = sizeof(size_t) * 2;
+constexpr size_t GAP_LOW_SIZE_OFFSET = sizeof(size_t) * 3;
+constexpr size_t GAP_HIGH_SIZE_OFFSET = sizeof(size_t);
+
+} // namespace flat_tlsf
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_COMMON_H
diff --git a/libc/src/__support/flat_tlsf/global.cpp b/libc/src/__support/flat_tlsf/global.cpp
new file mode 100644
index 0000000000000..021b9a4aed7b0
--- /dev/null
+++ b/libc/src/__support/flat_tlsf/global.cpp
@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide global FlatTlsfHeap instance.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/flat_tlsf/global.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace flat_tlsf {
+
+LIBC_CONSTINIT FlatTlsfHeap flat_tlsf_heap_symbols;
+FlatTlsfHeap *flat_tlsf_heap = &flat_tlsf_heap_symbols;
+
+} // namespace flat_tlsf
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/flat_tlsf/global.h b/libc/src/__support/flat_tlsf/global.h
new file mode 100644
index 0000000000000..d14fec35d318d
--- /dev/null
+++ b/libc/src/__support/flat_tlsf/global.h
@@ -0,0 +1,117 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide Global Heap class for flat_tlsf tracking lazy initialization.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_GLOBAL_H
+#define LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_GLOBAL_H
+
+#include "src/__support/CPP/cstddef.h"
+#include "src/__support/CPP/span.h"
+#include "src/__support/flat_tlsf/heap.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/math_extras.h"
+#include "src/string/memory_utils/inline_memset.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+extern "C" cpp::byte _end;
+extern "C" cpp::byte __llvm_libc_heap_limit;
+
+namespace flat_tlsf {
+
+class FlatTlsfHeap {
+public:
+ LIBC_INLINE constexpr FlatTlsfHeap()
+ : begin(&_end), end(&__llvm_libc_heap_limit) {}
+
+ LIBC_INLINE constexpr FlatTlsfHeap(cpp::span<cpp::byte> region)
+ : begin(region.data()), end(region.data() + region.size()) {}
+
+ LIBC_INLINE void *allocate(size_t size) {
+ if (heap.get_gap_list() == nullptr)
+ init();
+ return heap.malloc(size);
+ }
+
+ LIBC_INLINE void *aligned_allocate(size_t alignment, size_t size) {
+ if (size == 0)
+ return nullptr;
+
+ // The alignment must be an integral power of two.
+ if (!bit_utils::is_power_of_2(alignment))
+ return nullptr;
+
+ // The size parameter must be an integral multiple of alignment.
+ if (size % alignment != 0)
+ return nullptr;
+
+ if (heap.get_gap_list() == nullptr)
+ init();
+ return heap.aligned_alloc(alignment, size);
+ }
+
+ LIBC_INLINE void free(void *ptr) {
+ if (ptr != nullptr)
+ LIBC_ASSERT(is_valid_ptr(ptr) && "Invalid pointer");
+ heap.free(ptr);
+ }
+
+ LIBC_INLINE void *realloc(void *ptr, size_t size) {
+ if (heap.get_gap_list() == nullptr)
+ init();
+ if (ptr != nullptr)
+ LIBC_ASSERT(is_valid_ptr(ptr) && "Invalid pointer");
+ return heap.realloc(ptr, size);
+ }
+
+ LIBC_INLINE void *calloc(size_t num, size_t size) {
+ size_t bytes;
+ if (mul_overflow(num, size, bytes))
+ return nullptr;
+ void *ptr = allocate(bytes);
+ if (ptr != nullptr)
+ inline_memset(ptr, 0, bytes);
+ return ptr;
+ }
+
+ LIBC_INLINE cpp::span<cpp::byte> region() const { return {begin, end}; }
+
+private:
+ LIBC_INLINE void init() {
+ LIBC_ASSERT(heap.get_gap_list() == nullptr && "duplicate initialization");
+ Byte *res = heap.claim(reinterpret_cast<Byte *>(begin),
+ static_cast<size_t>(end - begin));
+ LIBC_ASSERT(res != nullptr && "Failed to claim memory for heap");
+ }
+
+ LIBC_INLINE bool is_valid_ptr(void *ptr) { return ptr >= begin && ptr < end; }
+
+ cpp::byte *begin;
+ cpp::byte *end;
+ Heap heap;
+};
+
+template <size_t BUFF_SIZE> class FlatTlsfHeapBuffer : public FlatTlsfHeap {
+public:
+ LIBC_INLINE constexpr FlatTlsfHeapBuffer() : FlatTlsfHeap{buffer}, buffer{} {}
+
+private:
+ cpp::byte buffer[BUFF_SIZE];
+};
+
+extern FlatTlsfHeap *flat_tlsf_heap;
+
+} // namespace flat_tlsf
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_GLOBAL_H
diff --git a/libc/src/__support/flat_tlsf/heap.h b/libc/src/__support/flat_tlsf/heap.h
new file mode 100644
index 0000000000000..3e9dbffc4c9c6
--- /dev/null
+++ b/libc/src/__support/flat_tlsf/heap.h
@@ -0,0 +1,504 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide Heap class for the flat_tlsf allocator.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_HEAP_H
+#define LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_HEAP_H
+
+#include "hdr/types/size_t.h"
+#include "src/__support/CPP/algorithm.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/flat_tlsf/binning.h"
+#include "src/__support/flat_tlsf/bit_utils.h"
+#include "src/__support/flat_tlsf/bitfield.h"
+#include "src/__support/flat_tlsf/chunk.h"
+#include "src/__support/flat_tlsf/common.h"
+#include "src/__support/flat_tlsf/node.h"
+#include "src/__support/flat_tlsf/tag.h"
+#include "src/__support/libc_assert.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+#include "src/string/memory_utils/inline_memcpy.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace flat_tlsf {
+
+class Heap {
+ BitField available = {};
+ Node **gap_list = nullptr;
+
+ struct SearchResult {
+ Byte *base;
+ Byte *end;
+ };
+
+ LIBC_INLINE cpp::optional<SearchResult>
+ full_search_bin(uint32_t bin, size_t required_size, size_t align_mask) {
+ for (Node *node = gap_list[bin]; node != nullptr; node = node->next) {
+ size_t size = chunk::read_word<size_t>(chunk::gap_node_to_size(node));
+
+ Byte *base = chunk::gap_node_to_base(node);
+ Byte *end = base + size;
+ Byte *aligned_base = bit_utils::align_up_by_mask(base, align_mask);
+ if (aligned_base + required_size <= end) {
+ deregister_gap(base, size);
+ if (base != aligned_base)
+ register_gap(base, aligned_base);
+ else
+ tag::clear_above_free(chunk::end_to_tag(base));
+ return SearchResult{aligned_base, end};
+ }
+ }
+
+ return cpp::nullopt;
+ }
+
+ LIBC_INLINE void register_gap(Byte *base, Byte *end) {
+ LIBC_ASSERT(chunk::is_chunk_size(base, end));
+
+ size_t size = static_cast<size_t>(end - base);
+ uint32_t bin = cpp::min(Binning::size_to_bin(size),
+ static_cast<uint32_t>(Binning::BIN_COUNT - 1));
+ Node **bin_ptr = &gap_list[bin];
+
+ if (*bin_ptr == nullptr) {
+ LIBC_ASSERT(!available.read_bit(bin));
+ available.set_bit(bin);
+ }
+
+ chunk::gap_base_to_node(base)->link_at(Node{*bin_ptr, bin_ptr});
+ chunk::write_word(chunk::gap_base_to_bin(base), bin);
+ chunk::write_word(chunk::gap_base_to_size(base), size);
+ chunk::write_word(chunk::gap_end_to_size_and_flag(end), size);
+
+ LIBC_ASSERT(*bin_ptr != nullptr);
+ }
+
+ LIBC_INLINE void deregister_gap(Byte *base, size_t size) {
+ LIBC_ASSERT(
+ gap_list[cpp::min(Binning::size_to_bin(size),
+ static_cast<uint32_t>(Binning::BIN_COUNT - 1))] !=
+ nullptr);
+
+ chunk::gap_base_to_node(base)->unlink();
+
+ uint32_t bin = chunk::read_word<uint32_t>(chunk::gap_base_to_bin(base));
+ if (gap_list[bin] == nullptr) {
+ LIBC_ASSERT(available.read_bit(bin));
+ available.clear_bit(bin);
+ }
+ }
+
+public:
+ LIBC_INLINE constexpr Heap() = default;
+
+ // Add an area to be managed by the heap
+ LIBC_INLINE Byte *claim(Byte *base, size_t size) {
+ Byte *heap_end =
+ chunk::align_down(bit_utils::saturating_ptr_add(base, size));
+ Byte *heap_base;
+ Byte *gap_base;
+
+ // Gap lists haven't been initialized
+ if (gap_list == nullptr) {
+ base = cpp::max(cpp::bit_cast<Byte *>(uintptr_t{1}), base);
+ heap_base = bit_utils::align_up_by(base, alignof(Node *));
+ size_t gap_list_size = sizeof(Node *) * Binning::BIN_COUNT;
+ gap_base = chunk::align_up(heap_base + gap_list_size + sizeof(Byte));
+
+ // If calculating gap_base overflowed OR the gap_base is higher than
+ // heap_end there isn't enough memory to allocate the metadata and cap it
+ // off with a tag
+ if (gap_base < heap_base || heap_end < gap_base)
+ return nullptr;
+
+ Byte tag = tag::ALLOCATED_FLAG;
+ if (gap_base < heap_end)
+ tag |= tag::ABOVE_FREE_FLAG;
+ chunk::write_word(chunk::end_to_tag(gap_base), tag);
+ gap_list = reinterpret_cast<Node **>(heap_base);
+ for (size_t i = 0; i < Binning::BIN_COUNT; ++i)
+ gap_list[i] = nullptr;
+ } else {
+ // Note that adding the header size and aligning up automatically dodges
+ // the possibility of claiming null, if `memory` started at null.
+ gap_base = chunk::align_up(base + sizeof(Byte));
+
+ // If calculating gap_base overflowed OR there isn't a CHUNK_UNIT between
+ // gap_base and heap_end, then there isn't enough memory to claim
+ if (gap_base + CHUNK_UNIT < base || heap_end < gap_base + CHUNK_UNIT)
+ return nullptr;
+
+ heap_base = chunk::end_to_tag(gap_base);
+ chunk::write_word(heap_base, static_cast<Byte>(tag::ALLOCATED_FLAG |
+ tag::ABOVE_FREE_FLAG |
+ tag::HEAP_BASE_FLAG));
+ }
+ if (gap_base < heap_end) {
+ register_gap(gap_base, heap_end);
+ }
+
+ return heap_end;
+ }
+
+ LIBC_INLINE Node *get_gap_list_head(uint32_t bin) const {
+ return gap_list[bin];
+ }
+ LIBC_INLINE Node **get_gap_list_ptr(uint32_t bin) const {
+ return &gap_list[bin];
+ }
+ LIBC_INLINE const BitField &get_available() const { return available; }
+ LIBC_INLINE Node **get_gap_list() const { return gap_list; }
+ LIBC_INLINE void test_deregister_gap(Byte *base, size_t size) {
+ deregister_gap(base, size);
+ }
+
+ LIBC_INLINE Byte *allocate(size_t required_size, size_t required_align) {
+ size_t required_chunk_size = chunk::required_chunk_size(required_size);
+ Byte *base = nullptr;
+ Byte *chunk_end = nullptr;
+
+ size_t bin = Binning::size_to_bin_ceil(
+ cpp::max(required_chunk_size, required_align));
+
+ if (bin >= Binning::BIN_COUNT - 1) {
+ if (available.read_bit(Binning::BIN_COUNT - 1)) {
+ if (auto result =
+ full_search_bin(Binning::BIN_COUNT - 1, required_chunk_size,
+ required_align - 1)) {
+ base = result->base;
+ chunk_end = result->end;
+ goto success;
+ }
+ }
+ return nullptr;
+ }
+
+ {
+ size_t bit = available.bit_scan_after(static_cast<uint32_t>(bin));
+ if (bit >= Binning::BIN_COUNT) {
+ if (available.read_bit(static_cast<uint32_t>(bin - 1))) {
+ if (auto result =
+ full_search_bin(static_cast<uint32_t>(bin - 1),
+ required_chunk_size, required_align - 1)) {
+ base = result->base;
+ chunk_end = result->end;
+ goto success;
+ }
+ }
+ return nullptr;
+ }
+
+ if (required_align <= CHUNK_UNIT) {
+ Node *node_ptr = gap_list[bit];
+ size_t size =
+ chunk::read_word<size_t>(chunk::gap_node_to_size(node_ptr));
+
+ LIBC_ASSERT(size >= required_chunk_size);
+ base = chunk::gap_node_to_base(node_ptr);
+ deregister_gap(base, size);
+ tag::clear_above_free(chunk::end_to_tag(base));
+ chunk_end = base + size;
+ goto success;
+ } else {
+ size_t align_mask = required_align - 1;
+ while (true) {
+ for (Node *node = gap_list[bit]; node != nullptr; node = node->next) {
+ size_t size =
+ chunk::read_word<size_t>(chunk::gap_node_to_size(node));
+ Byte *b = chunk::gap_node_to_base(node);
+ Byte *end = b + size;
+ Byte *aligned_base = bit_utils::align_up_by_mask(b, align_mask);
+ if (aligned_base + required_chunk_size <= end) {
+ deregister_gap(b, size);
+ if (b != aligned_base) {
+ register_gap(b, aligned_base);
+ } else {
+ tag::clear_above_free(chunk::end_to_tag(b));
+ }
+ base = aligned_base;
+ chunk_end = end;
+ goto success;
+ }
+ }
+
+ if (bit + 1 < Binning::BIN_COUNT ||
+ BitField::BITS > Binning::BIN_COUNT) {
+ bit = available.bit_scan_after(static_cast<uint32_t>(bit + 1));
+ if (bit < Binning::BIN_COUNT)
+ continue;
+ }
+
+ // Inlined: full_search_bin(bin - 1, required_chunk_size, align_mask)
+ for (Node *node = gap_list[bin - 1]; node != nullptr;
+ node = node->next) {
+ size_t size =
+ chunk::read_word<size_t>(chunk::gap_node_to_size(node));
+ Byte *b = chunk::gap_node_to_base(node);
+ Byte *end = b + size;
+ Byte *aligned_base = bit_utils::align_up_by_mask(b, align_mask);
+ if (aligned_base + required_chunk_size <= end) {
+ deregister_gap(b, size);
+ if (b != aligned_base)
+ register_gap(b, aligned_base);
+ else
+ tag::clear_above_free(chunk::end_to_tag(b));
+
+ base = aligned_base;
+ chunk_end = end;
+ goto success;
+ }
+ }
+
+ return nullptr;
+ }
+ }
+ }
+
+ success:
+ LIBC_ASSERT(chunk::align_down(base) == base);
+
+ Byte *end = base + required_chunk_size;
+ Byte tag = tag::ALLOCATED_FLAG;
+ if (end != chunk_end) {
+ register_gap(end, chunk_end);
+ tag |= tag::ABOVE_FREE_FLAG;
+ }
+
+ chunk::write_word(chunk::end_to_tag(end), tag);
+ return base;
+ }
+
+ LIBC_INLINE void deallocate(Byte *ptr, size_t required_size) {
+ Byte *chunk_base = ptr;
+ Byte *chunk_end = chunk::alloc_to_end(chunk_base, required_size);
+ Byte tag = chunk::read_word<Byte>(chunk::end_to_tag(chunk_end));
+
+ LIBC_ASSERT(tag::is_allocated(tag));
+ LIBC_ASSERT(chunk::is_chunk_size(chunk_base, chunk_end));
+ // Try to recombine with a gap below, if it's there.
+ // This gap is never the end of the heap, so we don't need to worry about
+ // the presence of an end flag.
+ Byte *below_tag_ptr = chunk::end_to_tag(chunk_base);
+ if (!tag::is_allocated(chunk::read_word<Byte>(below_tag_ptr))) {
+ size_t below_size =
+ chunk::read_word<size_t>(chunk::gap_end_to_size_and_flag(chunk_base));
+
+ Byte *below_base = chunk_base - below_size;
+ deregister_gap(below_base, below_size);
+ chunk_base = below_base;
+ } else {
+ tag::set_above_free(below_tag_ptr);
+ }
+
+ // Try to recombine with a gap above, if it's there.
+ // The end flag is never clobbered by this operation, so we can still read
+ // it later.
+ if (tag::is_above_free(tag)) {
+ LIBC_ASSERT(!tag::is_heap_end(tag));
+ size_t above_size =
+ chunk::read_word<size_t>(chunk::gap_base_to_size(chunk_end));
+ deregister_gap(chunk_end, above_size);
+ chunk_end += above_size;
+ }
+
+ register_gap(chunk_base, chunk_end);
+ }
+
+ LIBC_INLINE bool try_grow_in_place(Byte *ptr, size_t old_size,
+ size_t new_size) {
+ LIBC_ASSERT(new_size >= old_size);
+
+ Byte *old_end = chunk::alloc_to_end(ptr, old_size);
+ Byte *new_end = chunk::alloc_to_end(ptr, new_size);
+
+ if (old_end == new_end)
+ return true;
+
+ Byte old_tag = chunk::read_word<Byte>(chunk::end_to_tag(old_end));
+ LIBC_ASSERT(tag::is_allocated(old_tag));
+
+ if (tag::is_above_free(old_tag)) {
+ size_t above_size =
+ chunk::read_word<size_t>(chunk::gap_base_to_size(old_end));
+ Byte *above_end = old_end + above_size;
+
+ if (new_end <= above_end) {
+ deregister_gap(old_end, above_size);
+
+ if (new_end != above_end) {
+ register_gap(new_end, above_end);
+ chunk::write_word(
+ chunk::end_to_tag(new_end),
+ static_cast<Byte>(tag::ALLOCATED_FLAG | tag::ABOVE_FREE_FLAG));
+ } else {
+ chunk::write_word(chunk::end_to_tag(new_end),
+ static_cast<Byte>(tag::ALLOCATED_FLAG));
+ }
+
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ LIBC_INLINE void shrink_in_place(Byte *ptr, size_t old_size,
+ size_t new_size) {
+ LIBC_ASSERT(new_size != 0);
+ LIBC_ASSERT(new_size <= old_size);
+
+ Byte *chunk_end = chunk::alloc_to_end(ptr, old_size);
+ Byte *new_end = chunk::alloc_to_end(ptr, new_size);
+
+ if (new_end != chunk_end) {
+ Byte old_tag = chunk::read_word<Byte>(chunk::end_to_tag(chunk_end));
+
+ if (tag::is_above_free(old_tag)) {
+ size_t above_size =
+ chunk::read_word<size_t>(chunk::gap_base_to_size(chunk_end));
+ deregister_gap(chunk_end, above_size);
+ chunk_end += above_size;
+ }
+
+ register_gap(new_end, chunk_end);
+ chunk::write_word(
+ chunk::end_to_tag(new_end),
+ static_cast<Byte>(tag::ALLOCATED_FLAG | tag::ABOVE_FREE_FLAG));
+ }
+ }
+
+ LIBC_INLINE bool try_reallocate_in_place(Byte *ptr, size_t old_size,
+ size_t new_size) {
+ if (new_size > old_size) {
+ return try_grow_in_place(ptr, old_size, new_size);
+ } else if (new_size < old_size) {
+ shrink_in_place(ptr, old_size, new_size);
+ return true;
+ } else {
+ return true;
+ }
+ }
+
+ LIBC_INLINE Byte *reallocate(Byte *ptr, size_t old_size, size_t new_size,
+ size_t new_align) {
+ if (try_reallocate_in_place(ptr, old_size, new_size))
+ return ptr;
+ Byte *new_ptr = allocate(new_size, new_align);
+ if (new_ptr == nullptr)
+ return nullptr;
+ inline_memcpy(new_ptr, ptr, old_size);
+ deallocate(ptr, old_size);
+ return new_ptr;
+ }
+
+ static constexpr size_t HEADER_SIZE = sizeof(size_t);
+
+ LIBC_INLINE void *malloc(size_t size) {
+ return aligned_alloc(CHUNK_UNIT, size);
+ }
+
+ LIBC_INLINE void *aligned_alloc(size_t align, size_t size) {
+ if (size == 0)
+ return nullptr;
+
+ size_t header_align = alignof(size_t); // 8 bytes
+ size_t allocated_align = cpp::max(align, header_align);
+
+ size_t shift = (HEADER_SIZE + align - 1) & ~(align - 1);
+ size_t allocated_size = size + shift;
+
+ Byte *base_ptr = allocate(allocated_size, allocated_align);
+ if (base_ptr == nullptr)
+ return nullptr;
+
+ size_t actual_chunk_size = chunk::required_chunk_size(allocated_size);
+ Byte *user_ptr = base_ptr + shift;
+
+ size_t shift_exponent = static_cast<size_t>(cpp::countr_zero(shift));
+
+ size_t *header = reinterpret_cast<size_t *>(user_ptr - HEADER_SIZE);
+ *header = actual_chunk_size | (shift_exponent & 31);
+
+ return user_ptr;
+ }
+
+ LIBC_INLINE void free(void *ptr) {
+ if (ptr == nullptr)
+ return;
+
+ Byte *user_ptr = static_cast<Byte *>(ptr);
+ size_t *header = reinterpret_cast<size_t *>(user_ptr - HEADER_SIZE);
+ size_t header_val = *header;
+
+ size_t shift_exponent = header_val & 31;
+ LIBC_ASSERT(shift_exponent >= 3 && "Invalid or double freed pointer");
+ if (shift_exponent < 3)
+ return;
+
+ size_t shift = size_t{1} << shift_exponent;
+ size_t actual_chunk_size = header_val & ~31;
+
+ Byte *base_ptr = user_ptr - shift;
+
+ deallocate(base_ptr, actual_chunk_size - 1);
+ }
+
+ LIBC_INLINE void *realloc(void *ptr, size_t new_size) {
+ if (ptr == nullptr)
+ return malloc(new_size);
+
+ if (new_size == 0) {
+ free(ptr);
+ return nullptr;
+ }
+
+ Byte *user_ptr = static_cast<Byte *>(ptr);
+ size_t *header = reinterpret_cast<size_t *>(user_ptr - HEADER_SIZE);
+ size_t header_val = *header;
+
+ size_t shift_exponent = header_val & 31;
+ if (shift_exponent < 3)
+ return nullptr;
+
+ size_t shift = size_t{1} << shift_exponent;
+ size_t old_chunk_size = header_val & ~31;
+
+ Byte *base_ptr = user_ptr - shift;
+
+ size_t new_allocated_size = new_size + shift;
+ size_t new_chunk_size = chunk::required_chunk_size(new_allocated_size);
+
+ if (try_reallocate_in_place(base_ptr, old_chunk_size - 1,
+ new_chunk_size - 1)) {
+ *header = new_chunk_size | shift_exponent;
+ return user_ptr;
+ }
+
+ Byte *new_user_ptr = static_cast<Byte *>(aligned_alloc(shift, new_size));
+ if (new_user_ptr == nullptr)
+ return nullptr;
+
+ size_t old_user_size = old_chunk_size - shift - 1;
+ size_t bytes_to_copy = cpp::min(old_user_size, new_size);
+ inline_memcpy(new_user_ptr, user_ptr, bytes_to_copy);
+
+ free(ptr);
+ return new_user_ptr;
+ }
+};
+
+} // namespace flat_tlsf
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_HEAP_H
diff --git a/libc/src/__support/flat_tlsf/node.h b/libc/src/__support/flat_tlsf/node.h
new file mode 100644
index 0000000000000..0664fedfb1eee
--- /dev/null
+++ b/libc/src/__support/flat_tlsf/node.h
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide Node struct for the flat_tlsf allocator.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_NODE_H
+#define LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_NODE_H
+
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace flat_tlsf {
+
+struct Node {
+ Node *next;
+ // Use next_of_prev to avoid branches on special guardian pointer.
+ Node **next_of_prev;
+
+ LIBC_INLINE Node **addr_of_next() { return &next; }
+
+ LIBC_INLINE void link_at(Node data) {
+ *this = data;
+ *data.next_of_prev = this;
+ if (data.next)
+ data.next->next_of_prev = addr_of_next();
+ }
+
+ LIBC_INLINE void unlink() {
+ *next_of_prev = next;
+ if (next)
+ next->next_of_prev = next_of_prev;
+ }
+};
+
+} // namespace flat_tlsf
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_NODE_H
diff --git a/libc/src/__support/flat_tlsf/tag.h b/libc/src/__support/flat_tlsf/tag.h
new file mode 100644
index 0000000000000..7feb540e22932
--- /dev/null
+++ b/libc/src/__support/flat_tlsf/tag.h
@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide tag utilities for the flat_tlsf allocator.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_TAG_H
+#define LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_TAG_H
+
+#include "src/__support/flat_tlsf/common.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace flat_tlsf {
+namespace tag {
+
+static constexpr Byte ALLOCATED_FLAG = 0b0001;
+static constexpr Byte ABOVE_FREE_FLAG = 0b0010;
+static constexpr Byte HEAP_BASE_FLAG = 0b0100;
+static constexpr Byte HEAP_END_FLAG = 0b1000;
+
+LIBC_INLINE bool is_above_free(Byte tag) { return tag & ABOVE_FREE_FLAG; }
+
+LIBC_INLINE bool is_allocated(Byte tag) { return tag & ALLOCATED_FLAG; }
+
+LIBC_INLINE bool is_heap_base(Byte tag) { return tag & HEAP_BASE_FLAG; }
+
+LIBC_INLINE bool is_heap_end(Byte tag) { return tag & HEAP_END_FLAG; }
+
+LIBC_INLINE void set_above_free(Byte *ptr) { *ptr |= ABOVE_FREE_FLAG; }
+
+LIBC_INLINE void clear_above_free(Byte *ptr) { *ptr ^= ABOVE_FREE_FLAG; }
+
+LIBC_INLINE void set_end_flag(Byte *ptr) { *ptr ^= HEAP_END_FLAG; }
+
+LIBC_INLINE void clear_end_flag(Byte *ptr) { *ptr ^= HEAP_END_FLAG; }
+
+} // namespace tag
+} // namespace flat_tlsf
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_TAG_H
diff --git a/libc/src/__support/heap.cpp b/libc/src/__support/heap.cpp
new file mode 100644
index 0000000000000..c798fb4dd22ca
--- /dev/null
+++ b/libc/src/__support/heap.cpp
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide global_heap pointer definition and initialization.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/heap.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+#if defined(LIBC_HEAP_TYPE_FLAT_TLSF)
+namespace flat_tlsf {
+extern FlatTlsfHeap flat_tlsf_heap_symbols;
+} // namespace flat_tlsf
+flat_tlsf::FlatTlsfHeap *global_heap = &flat_tlsf::flat_tlsf_heap_symbols;
+#else
+extern FreeListHeap freelist_heap_symbols;
+FreeListHeap *global_heap = &freelist_heap_symbols;
+#endif
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/heap.h b/libc/src/__support/heap.h
new file mode 100644
index 0000000000000..9d2d8f90e4eda
--- /dev/null
+++ b/libc/src/__support/heap.h
@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provide a generic Heap interface to switch between allocator
+/// implementations.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_HEAP_H
+#define LLVM_LIBC_SRC___SUPPORT_HEAP_H
+
+#include "src/__support/macros/config.h"
+
+#if defined(LIBC_HEAP_TYPE_FLAT_TLSF)
+#include "src/__support/flat_tlsf/global.h"
+#else
+#include "src/__support/freelist_heap.h"
+#endif
+
+namespace LIBC_NAMESPACE_DECL {
+
+#if defined(LIBC_HEAP_TYPE_FLAT_TLSF)
+using GlobalHeap = flat_tlsf::FlatTlsfHeap;
+extern flat_tlsf::FlatTlsfHeap *global_heap;
+#else
+using GlobalHeap = FreeListHeap;
+extern FreeListHeap *global_heap;
+#endif
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_HEAP_H
diff --git a/libc/src/stdlib/baremetal/CMakeLists.txt b/libc/src/stdlib/baremetal/CMakeLists.txt
index 2193ed44612d9..d63a9cf237dc2 100644
--- a/libc/src/stdlib/baremetal/CMakeLists.txt
+++ b/libc/src/stdlib/baremetal/CMakeLists.txt
@@ -11,7 +11,7 @@ add_entrypoint_object(
HDRS
../malloc.h
DEPENDS
- libc.src.__support.freelist_heap
+ libc.src.__support.heap
)
add_entrypoint_object(
@@ -21,7 +21,7 @@ add_entrypoint_object(
HDRS
../free.h
DEPENDS
- libc.src.__support.freelist_heap
+ libc.src.__support.heap
)
add_entrypoint_object(
@@ -31,7 +31,7 @@ add_entrypoint_object(
HDRS
../calloc.h
DEPENDS
- libc.src.__support.freelist_heap
+ libc.src.__support.heap
)
add_entrypoint_object(
@@ -41,7 +41,7 @@ add_entrypoint_object(
HDRS
../realloc.h
DEPENDS
- libc.src.__support.freelist_heap
+ libc.src.__support.heap
)
add_entrypoint_object(
@@ -51,5 +51,5 @@ add_entrypoint_object(
HDRS
../aligned_alloc.h
DEPENDS
- libc.src.__support.freelist_heap
+ libc.src.__support.heap
)
diff --git a/libc/src/stdlib/baremetal/aligned_alloc.cpp b/libc/src/stdlib/baremetal/aligned_alloc.cpp
index e9548719c3a63..5def1bf29a17c 100644
--- a/libc/src/stdlib/baremetal/aligned_alloc.cpp
+++ b/libc/src/stdlib/baremetal/aligned_alloc.cpp
@@ -7,7 +7,7 @@
//===----------------------------------------------------------------------===//
#include "src/stdlib/aligned_alloc.h"
-#include "src/__support/freelist_heap.h"
+#include "src/__support/heap.h"
#include "src/__support/macros/config.h"
#include <stddef.h>
@@ -15,7 +15,7 @@
namespace LIBC_NAMESPACE_DECL {
LLVM_LIBC_FUNCTION(void *, aligned_alloc, (size_t alignment, size_t size)) {
- return freelist_heap->aligned_allocate(alignment, size);
+ return global_heap->aligned_allocate(alignment, size);
}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/baremetal/calloc.cpp b/libc/src/stdlib/baremetal/calloc.cpp
index 2b3b83cebc8ac..67f4aae8b0441 100644
--- a/libc/src/stdlib/baremetal/calloc.cpp
+++ b/libc/src/stdlib/baremetal/calloc.cpp
@@ -7,7 +7,7 @@
//===----------------------------------------------------------------------===//
#include "src/stdlib/calloc.h"
-#include "src/__support/freelist_heap.h"
+#include "src/__support/heap.h"
#include "src/__support/macros/config.h"
#include <stddef.h>
@@ -15,7 +15,7 @@
namespace LIBC_NAMESPACE_DECL {
LLVM_LIBC_FUNCTION(void *, calloc, (size_t num, size_t size)) {
- return freelist_heap->calloc(num, size);
+ return global_heap->calloc(num, size);
}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/baremetal/free.cpp b/libc/src/stdlib/baremetal/free.cpp
index 1e25fe5f2dcfe..5d89e730d79e4 100644
--- a/libc/src/stdlib/baremetal/free.cpp
+++ b/libc/src/stdlib/baremetal/free.cpp
@@ -7,13 +7,13 @@
//===----------------------------------------------------------------------===//
#include "src/stdlib/free.h"
-#include "src/__support/freelist_heap.h"
+#include "src/__support/heap.h"
#include "src/__support/macros/config.h"
#include <stddef.h>
namespace LIBC_NAMESPACE_DECL {
-LLVM_LIBC_FUNCTION(void, free, (void *ptr)) { return freelist_heap->free(ptr); }
+LLVM_LIBC_FUNCTION(void, free, (void *ptr)) { return global_heap->free(ptr); }
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/baremetal/malloc.cpp b/libc/src/stdlib/baremetal/malloc.cpp
index a299282667fcd..6bc87533064a7 100644
--- a/libc/src/stdlib/baremetal/malloc.cpp
+++ b/libc/src/stdlib/baremetal/malloc.cpp
@@ -7,7 +7,7 @@
//===----------------------------------------------------------------------===//
#include "src/stdlib/malloc.h"
-#include "src/__support/freelist_heap.h"
+#include "src/__support/heap.h"
#include "src/__support/macros/config.h"
#include <stddef.h>
@@ -15,7 +15,7 @@
namespace LIBC_NAMESPACE_DECL {
LLVM_LIBC_FUNCTION(void *, malloc, (size_t size)) {
- return freelist_heap->allocate(size);
+ return global_heap->allocate(size);
}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/baremetal/realloc.cpp b/libc/src/stdlib/baremetal/realloc.cpp
index fb25c68ec4296..c8708814db275 100644
--- a/libc/src/stdlib/baremetal/realloc.cpp
+++ b/libc/src/stdlib/baremetal/realloc.cpp
@@ -7,7 +7,7 @@
//===----------------------------------------------------------------------===//
#include "src/stdlib/realloc.h"
-#include "src/__support/freelist_heap.h"
+#include "src/__support/heap.h"
#include "src/__support/macros/config.h"
#include <stddef.h>
@@ -15,7 +15,7 @@
namespace LIBC_NAMESPACE_DECL {
LLVM_LIBC_FUNCTION(void *, realloc, (void *ptr, size_t size)) {
- return freelist_heap->realloc(ptr, size);
+ return global_heap->realloc(ptr, size);
}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 6b9c1b4ac8cc7..0749e468eb845 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -308,6 +308,7 @@ add_subdirectory(RPC)
add_subdirectory(OSUtil)
add_subdirectory(FPUtil)
add_subdirectory(fixed_point)
+add_subdirectory(flat_tlsf)
add_subdirectory(HashTable)
add_subdirectory(time)
add_subdirectory(threads)
diff --git a/libc/test/src/__support/flat_tlsf/CMakeLists.txt b/libc/test/src/__support/flat_tlsf/CMakeLists.txt
new file mode 100644
index 0000000000000..812e579493feb
--- /dev/null
+++ b/libc/test/src/__support/flat_tlsf/CMakeLists.txt
@@ -0,0 +1,97 @@
+add_custom_target(libc-flat-tlsf-tests)
+
+add_libc_test(
+ bitfield_test
+ SUITE
+ libc-flat-tlsf-tests
+ SRCS
+ bitfield_test.cpp
+ DEPENDS
+ libc.src.__support.flat_tlsf.bitfield
+ libc.src.__support.flat_tlsf.bit_utils
+ libc.src.__support.CPP.limits
+ libc.src.__support.CPP.algorithm
+)
+
+add_libc_test(
+ binning_test
+ SUITE
+ libc-flat-tlsf-tests
+ SRCS
+ binning_test.cpp
+ DEPENDS
+ libc.src.__support.flat_tlsf.binning
+ libc.src.__support.flat_tlsf.common
+ libc.src.__support.CPP.limits
+ libc.src.__support.CPP.algorithm
+ libc.src.__support.CPP.array
+ libc.src.__support.CPP.optional
+)
+
+add_libc_test(
+ tag_test
+ SUITE
+ libc-flat-tlsf-tests
+ SRCS
+ tag_test.cpp
+ DEPENDS
+ libc.src.__support.flat_tlsf.tag
+ libc.src.__support.flat_tlsf.common
+)
+
+add_libc_test(
+ node_test
+ SUITE
+ libc-flat-tlsf-tests
+ SRCS
+ node_test.cpp
+ DEPENDS
+ libc.src.__support.flat_tlsf.node
+)
+
+add_libc_test(
+ chunk_test
+ SUITE
+ libc-flat-tlsf-tests
+ SRCS
+ chunk_test.cpp
+ DEPENDS
+ libc.src.__support.flat_tlsf.chunk
+ libc.src.__support.flat_tlsf.common
+ libc.src.__support.flat_tlsf.node
+)
+
+add_libc_test(
+ heap_test
+ SUITE
+ libc-flat-tlsf-tests
+ SRCS
+ heap_test.cpp
+ DEPENDS
+ libc.src.__support.flat_tlsf.heap
+ libc.src.__support.flat_tlsf.common
+ libc.src.__support.flat_tlsf.node
+ libc.src.__support.flat_tlsf.binning
+ libc.src.__support.flat_tlsf.chunk
+ libc.src.__support.flat_tlsf.tag
+ libc.src.__support.CPP.limits
+ libc.src.__support.CPP.algorithm
+ libc.src.string.memory_utils.inline_memset
+)
+
+if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
+ add_libc_test(
+ global_test
+ SUITE
+ libc-flat-tlsf-tests
+ SRCS
+ global_test.cpp
+ DEPENDS
+ libc.src.__support.flat_tlsf.global
+ libc.src.__support.flat_tlsf.common
+ libc.src.__support.CPP.span
+ libc.src.__support.CPP.new
+ libc.src.string.memory_utils.inline_memcpy
+ libc.src.string.memory_utils.inline_memset
+ )
+endif()
diff --git a/libc/test/src/__support/flat_tlsf/binning_test.cpp b/libc/test/src/__support/flat_tlsf/binning_test.cpp
new file mode 100644
index 0000000000000..afae9b6d5df55
--- /dev/null
+++ b/libc/test/src/__support/flat_tlsf/binning_test.cpp
@@ -0,0 +1,150 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for flat_tlsf binning.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/algorithm.h"
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/flat_tlsf/binning.h"
+#include "src/__support/flat_tlsf/common.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::cpp::array;
+using LIBC_NAMESPACE::cpp::max;
+using LIBC_NAMESPACE::cpp::nullopt;
+using LIBC_NAMESPACE::cpp::numeric_limits;
+using LIBC_NAMESPACE::cpp::optional;
+using LIBC_NAMESPACE::flat_tlsf::Binning;
+using LIBC_NAMESPACE::flat_tlsf::CHUNK_UNIT;
+
+namespace {
+
+class LlvmLibcFlatTlsfBinningTest : public LIBC_NAMESPACE::testing::Test {
+protected:
+ template <typename F>
+ size_t find_binning_boundary(uint32_t next_bin, size_t base, size_t end,
+ F &&size_to_bin) {
+ while (base < end) {
+ size_t mid = base + (end - base) / 2;
+ if (size_to_bin(mid) >= next_bin)
+ end = mid;
+ else
+ base = mid + 1;
+ }
+ return base;
+ }
+
+ template <typename F1, typename F2>
+ void find_binning_boundaries(size_t start_from_size,
+ optional<uint32_t> stop_at_bin, F1 &&size_to_bin,
+ F2 &&bin_boundary_callback) {
+ size_t prev_size = start_from_size;
+ size_t size = start_from_size;
+ size_t increment = 1;
+
+ optional<uint32_t> prev_bin;
+
+ while (true) {
+ uint32_t bin = size_to_bin(size);
+
+ if (!prev_bin.has_value() || prev_bin.value() != bin) {
+ if (prev_bin.has_value())
+ size = find_binning_boundary(prev_bin.value() + 1, prev_size, size,
+ size_to_bin);
+
+ bin_boundary_callback(bin, size);
+
+ increment = max<size_t>((size - prev_size) / 4, 1);
+ prev_size = size;
+
+ prev_bin = optional<uint32_t>(bin);
+ }
+
+ if (size != numeric_limits<size_t>::max()) {
+ if (size <= numeric_limits<size_t>::max() - increment)
+ size += increment;
+ else
+ size = numeric_limits<size_t>::max();
+ } else {
+ break;
+ }
+
+ if (stop_at_bin.has_value() && stop_at_bin.value() == bin)
+ break;
+ }
+ }
+
+ template <typename F>
+ void check_binning_properties(optional<uint32_t> stop_at_bin,
+ F &&size_to_bin) {
+ optional<uint32_t> prev_bin;
+
+ auto callback = [&](uint32_t bin, size_t size) {
+ if (prev_bin.has_value())
+ EXPECT_EQ(prev_bin.value() + 1, bin);
+ prev_bin = optional<uint32_t>(bin);
+
+ EXPECT_TRUE(
+ !stop_at_bin.has_value() || bin <= stop_at_bin.value() ||
+ (bin == numeric_limits<uint32_t>::max() && size < CHUNK_UNIT));
+ };
+
+ find_binning_boundaries(CHUNK_UNIT - 1, stop_at_bin, size_to_bin, callback);
+ }
+};
+
+TEST_F(LlvmLibcFlatTlsfBinningTest, CheckFindBinningBoundary) {
+ array<uint32_t, 12> size_to_bin = {0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 5};
+ auto size_to_bin_fn = [&](size_t s) { return size_to_bin[s]; };
+
+ EXPECT_EQ(find_binning_boundary(2, 1, 3, size_to_bin_fn), size_t{3});
+ EXPECT_EQ(find_binning_boundary(2, 3, 5, size_to_bin_fn), size_t{4});
+ EXPECT_EQ(find_binning_boundary(2, 2, 4, size_to_bin_fn), size_t{4});
+ EXPECT_EQ(find_binning_boundary(2, 4, 6, size_to_bin_fn), size_t{4});
+ EXPECT_EQ(find_binning_boundary(2, 5, 7, size_to_bin_fn), size_t{5});
+
+ EXPECT_EQ(find_binning_boundary(2, 2, 11, size_to_bin_fn), size_t{4});
+ EXPECT_EQ(find_binning_boundary(2, 0, 7, size_to_bin_fn), size_t{4});
+
+ EXPECT_EQ(find_binning_boundary(4, 0, 11, size_to_bin_fn), size_t{11});
+}
+
+TEST_F(LlvmLibcFlatTlsfBinningTest, CheckFindBinningBoundaries) {
+ array<uint32_t, 12> size_to_bin = {0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3};
+ array<size_t, 4> boundary_sizes = {0, 1, 4, 10};
+
+ size_t i = 0;
+ auto verifier = [&](uint32_t bin, size_t size) {
+ ASSERT_LT(i, boundary_sizes.size());
+ EXPECT_EQ(size, boundary_sizes[i]);
+ EXPECT_EQ(bin, size_to_bin[size]);
+ i++;
+ };
+
+ auto size_to_bin_fn = [&](size_t s) { return size_to_bin[s]; };
+
+ find_binning_boundaries(0, 3, size_to_bin_fn, verifier);
+ EXPECT_EQ(i, size_t{4});
+}
+
+TEST_F(LlvmLibcFlatTlsfBinningTest,
+ TestLinearExtentThenLinearlyDividedExponentialBinning) {
+ auto size_to_bin_fn = [](size_t size) {
+ return static_cast<uint32_t>(
+ Binning::linear_extend_then_linearly_divided_expotential_binning<8, 4>(
+ size));
+ };
+ check_binning_properties(nullopt, size_to_bin_fn);
+}
+
+} // namespace
diff --git a/libc/test/src/__support/flat_tlsf/bitfield_test.cpp b/libc/test/src/__support/flat_tlsf/bitfield_test.cpp
new file mode 100644
index 0000000000000..78b8fc7c1d0ef
--- /dev/null
+++ b/libc/test/src/__support/flat_tlsf/bitfield_test.cpp
@@ -0,0 +1,195 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for flat_tlsf bit utilities and BitField.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/algorithm.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/flat_tlsf/bit_utils.h"
+#include "src/__support/flat_tlsf/bitfield.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::cpp::numeric_limits;
+using LIBC_NAMESPACE::flat_tlsf::BitField;
+using LIBC_NAMESPACE::flat_tlsf::Byte;
+namespace bit_utils = LIBC_NAMESPACE::flat_tlsf::bit_utils;
+
+namespace {
+
+#define EXPECT_BF_EQ(lhs, rhs) \
+ for (size_t _i = 0; _i < BitField::NUMBER_OF_ELEMENTS; ++_i) { \
+ EXPECT_EQ((lhs).storage[_i], (rhs).storage[_i]); \
+ }
+
+TEST(LlvmLibcFlatTlsfTest, BitUtilsPointerAlignment) {
+ alignas(64) Byte bytes[128] = {};
+ Byte *ptr = bytes + 37;
+
+ EXPECT_TRUE(bit_utils::is_aligned_to(bytes, 64));
+ EXPECT_FALSE(bit_utils::is_aligned_to(ptr, 32));
+ EXPECT_EQ(bit_utils::align_down_by(ptr, 32), bytes + 32);
+ EXPECT_EQ(bit_utils::align_up_by(ptr, 32), bytes + 64);
+ EXPECT_EQ(bit_utils::align_up_by_mask(ptr, 31), bytes + 64);
+ EXPECT_EQ(bit_utils::align_up_by(bytes + 64, 32), bytes + 64);
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitUtilsSaturatingPtrAdd) {
+ Byte *ptr = reinterpret_cast<Byte *>(uintptr_t{100});
+ EXPECT_EQ(bit_utils::saturating_ptr_add(ptr, 23),
+ reinterpret_cast<Byte *>(uintptr_t{123}));
+
+ Byte *near_end =
+ reinterpret_cast<Byte *>(numeric_limits<uintptr_t>::max() - uintptr_t{3});
+ EXPECT_EQ(bit_utils::saturating_ptr_add(near_end, 4),
+ reinterpret_cast<Byte *>(numeric_limits<uintptr_t>::max()));
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldTestBitScanAfter) {
+ BitField field = BitField::zeros();
+ field.storage[0] = 0b10100;
+ field.storage[1] = 0b00010;
+
+ EXPECT_EQ(field.bit_scan_after(0), 2u);
+ EXPECT_EQ(field.bit_scan_after(2), 2u);
+ EXPECT_EQ(field.bit_scan_after(3), 4u);
+ EXPECT_EQ(field.bit_scan_after(4), 4u);
+ EXPECT_EQ(field.bit_scan_after(5),
+ static_cast<uint32_t>(BitField::BITS_PER_ELEMENT + 1));
+ EXPECT_EQ(field.bit_scan_after(BitField::BITS_PER_ELEMENT + 1),
+ static_cast<uint32_t>(BitField::BITS_PER_ELEMENT + 1));
+ EXPECT_EQ(field.bit_scan_after(BitField::BITS_PER_ELEMENT + 2),
+ static_cast<uint32_t>(BitField::BITS));
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldSetUnset) {
+ for (uint32_t i = 0; i < BitField::BITS; ++i) {
+ BitField bf = BitField::zeros();
+ bf.set_bit(i);
+ bf.clear_bit(i);
+ EXPECT_BF_EQ(bf, BitField::zeros());
+ }
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldSetEqSetAllUnsetRest) {
+ for (uint32_t i = 0; i < BitField::BITS; ++i) {
+ BitField bf = BitField::zeros();
+ for (uint32_t j = 0; j < i; ++j)
+ bf.set_bit(j);
+
+ BitField bf2 = BitField::zeros();
+ for (uint32_t j = 0; j < BitField::BITS; ++j)
+ bf2.set_bit(j);
+ for (uint32_t j = i; j < BitField::BITS; ++j)
+ bf2.clear_bit(j);
+
+ EXPECT_BF_EQ(bf, bf2);
+ }
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldBsfZero) {
+ BitField bf = BitField::zeros();
+ for (uint32_t i = 0; i < BitField::BITS; ++i)
+ EXPECT_EQ(bf.bit_scan_after(i), static_cast<uint32_t>(BitField::BITS));
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldBsf) {
+ for (uint32_t i = 0; i < BitField::BITS; ++i) {
+ BitField bf = BitField::zeros();
+ bf.set_bit(i);
+ EXPECT_EQ(bf.bit_scan_after(0), i);
+ }
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldBsfFromIndex) {
+ for (uint32_t i = 0; i < BitField::BITS; ++i) {
+ BitField bf = BitField::zeros();
+
+ for (uint32_t j = i; j < BitField::BITS; ++j)
+ bf.set_bit(j);
+
+ for (uint32_t j = 0; j < BitField::BITS; ++j)
+ EXPECT_EQ(bf.bit_scan_after(j), LIBC_NAMESPACE::cpp::max(i, j));
+ }
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldBsfFromIndex1) {
+ BitField bf = BitField::zeros();
+ bf.set_bit(0);
+ for (uint32_t i = 1; i < BitField::BITS; ++i)
+ EXPECT_EQ(bf.bit_scan_after(i), static_cast<uint32_t>(BitField::BITS));
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldBsfFirstLast) {
+ BitField bf = BitField::zeros();
+ bf.set_bit(0);
+ bf.set_bit(BitField::BITS - 1);
+
+ EXPECT_EQ(bf.bit_scan_after(0), 0u);
+ for (uint32_t i = 1; i < BitField::BITS; ++i)
+ EXPECT_EQ(bf.bit_scan_after(i), static_cast<uint32_t>(BitField::BITS - 1));
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldBsfOneBehind) {
+ for (uint32_t i = 1; i < BitField::BITS; ++i) {
+ BitField bf = BitField::zeros();
+ bf.set_bit(i - 1);
+ EXPECT_EQ(bf.bit_scan_after(i), static_cast<uint32_t>(BitField::BITS));
+ }
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldBsfOneBehindOneForward) {
+ for (uint32_t i = 1; i < BitField::BITS - 1; ++i) {
+ BitField bf = BitField::zeros();
+ bf.set_bit(i - 1);
+ bf.set_bit(i + 1);
+ EXPECT_EQ(bf.bit_scan_after(i), i + 1);
+ }
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldBsfOneBehindOneForwardOneOnPoint) {
+ for (uint32_t i = 1; i < BitField::BITS - 1; ++i) {
+ BitField bf = BitField::zeros();
+ bf.set_bit(i - 1);
+ bf.set_bit(i);
+ bf.set_bit(i + 1);
+ EXPECT_EQ(bf.bit_scan_after(i), i);
+ }
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldBsfOneForward) {
+ for (uint32_t i = 0; i < BitField::BITS - 1; ++i) {
+ BitField bf = BitField::zeros();
+ bf.set_bit(i + 1);
+ EXPECT_EQ(bf.bit_scan_after(i), i + 1);
+ }
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldBsfOnes) {
+ for (uint32_t i = 0; i < BitField::BITS; ++i) {
+ BitField bf = BitField::zeros();
+
+ for (uint32_t j = i; j < BitField::BITS; ++j)
+ bf.set_bit(j);
+ EXPECT_EQ(bf.bit_scan_after(0), i);
+ }
+}
+
+TEST(LlvmLibcFlatTlsfTest, BitFieldBsfOnesBelow) {
+ for (uint32_t i = 0; i < BitField::BITS; ++i) {
+ BitField bf = BitField::zeros();
+ for (uint32_t j = 0; j < i; ++j)
+ bf.set_bit(j);
+
+ EXPECT_EQ(bf.bit_scan_after(i), static_cast<uint32_t>(BitField::BITS));
+ }
+}
+
+} // namespace
diff --git a/libc/test/src/__support/flat_tlsf/chunk_test.cpp b/libc/test/src/__support/flat_tlsf/chunk_test.cpp
new file mode 100644
index 0000000000000..8220952ac023c
--- /dev/null
+++ b/libc/test/src/__support/flat_tlsf/chunk_test.cpp
@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for flat_tlsf chunk utilities.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/flat_tlsf/chunk.h"
+#include "src/__support/flat_tlsf/common.h"
+#include "src/__support/flat_tlsf/node.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::flat_tlsf::Byte;
+using LIBC_NAMESPACE::flat_tlsf::CHUNK_UNIT;
+using LIBC_NAMESPACE::flat_tlsf::GAP_BIN_OFFSET;
+using LIBC_NAMESPACE::flat_tlsf::GAP_HIGH_SIZE_OFFSET;
+using LIBC_NAMESPACE::flat_tlsf::GAP_LOW_SIZE_OFFSET;
+using LIBC_NAMESPACE::flat_tlsf::GAP_NODE_OFFSET;
+using LIBC_NAMESPACE::flat_tlsf::Node;
+namespace chunk = LIBC_NAMESPACE::flat_tlsf::chunk;
+
+TEST(LlvmLibcFlatTlsfChunkTest, GapPointerConversions) {
+ alignas(CHUNK_UNIT) Byte bytes[CHUNK_UNIT * 2] = {};
+ Byte *base = bytes;
+ Byte *end = bytes + CHUNK_UNIT;
+
+ EXPECT_EQ(reinterpret_cast<Byte *>(chunk::gap_base_to_node(base)),
+ base + GAP_NODE_OFFSET);
+ EXPECT_EQ(reinterpret_cast<Byte *>(chunk::gap_base_to_bin(base)),
+ base + GAP_BIN_OFFSET);
+ EXPECT_EQ(reinterpret_cast<Byte *>(chunk::gap_base_to_size(base)),
+ base + GAP_LOW_SIZE_OFFSET);
+ EXPECT_EQ(reinterpret_cast<Byte *>(chunk::gap_end_to_size_and_flag(end)),
+ end - GAP_HIGH_SIZE_OFFSET);
+
+ Node *node = chunk::gap_base_to_node(base);
+ EXPECT_EQ(chunk::gap_node_to_base(node), base);
+ EXPECT_EQ(reinterpret_cast<Byte *>(chunk::gap_node_to_size(node)),
+ base + GAP_LOW_SIZE_OFFSET);
+ EXPECT_EQ(chunk::end_to_tag(end), end - sizeof(Byte));
+}
+
+TEST(LlvmLibcFlatTlsfChunkTest, AlignsByChunkUnit) {
+ alignas(CHUNK_UNIT) Byte bytes[CHUNK_UNIT * 3] = {};
+ Byte *ptr = bytes + CHUNK_UNIT + 1;
+
+ EXPECT_EQ(chunk::align_down(ptr), bytes + CHUNK_UNIT);
+ EXPECT_EQ(chunk::align_up(ptr), bytes + CHUNK_UNIT * 2);
+ EXPECT_EQ(chunk::align_up(bytes + CHUNK_UNIT), bytes + CHUNK_UNIT);
+}
+
+TEST(LlvmLibcFlatTlsfChunkTest, RequiredChunkSize) {
+ // Size + 1 (for tag) rounded up to CHUNK_UNIT
+ EXPECT_EQ(chunk::required_chunk_size(0), CHUNK_UNIT);
+ EXPECT_EQ(chunk::required_chunk_size(CHUNK_UNIT - 1), CHUNK_UNIT);
+ EXPECT_EQ(chunk::required_chunk_size(CHUNK_UNIT), CHUNK_UNIT * 2);
+ EXPECT_EQ(chunk::required_chunk_size(CHUNK_UNIT + 5), CHUNK_UNIT * 2);
+}
+
+TEST(LlvmLibcFlatTlsfChunkTest, AllocToEnd) {
+ Byte *base = reinterpret_cast<Byte *>(uintptr_t{0x1000});
+ EXPECT_EQ(chunk::alloc_to_end(base, CHUNK_UNIT), base + CHUNK_UNIT * 2);
+}
+
+TEST(LlvmLibcFlatTlsfChunkTest, ReadWriteWord) {
+ uint64_t val = 0x0123456789abcdefULL;
+ Byte buffer[8] = {};
+
+ chunk::write_word(buffer, val);
+ EXPECT_EQ(chunk::read_word<uint64_t>(buffer), val);
+}
diff --git a/libc/test/src/__support/flat_tlsf/global_test.cpp b/libc/test/src/__support/flat_tlsf/global_test.cpp
new file mode 100644
index 0000000000000..c19d1f2e0366b
--- /dev/null
+++ b/libc/test/src/__support/flat_tlsf/global_test.cpp
@@ -0,0 +1,286 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for flat_tlsf global heap.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/new.h"
+#include "src/__support/CPP/span.h"
+#include "src/__support/flat_tlsf/common.h"
+#include "src/__support/flat_tlsf/global.h"
+#include "src/__support/macros/config.h"
+#include "src/string/memory_utils/inline_memcpy.h"
+#include "src/string/memory_utils/inline_memset.h"
+#include "test/UnitTest/Test.h"
+#include <stddef.h> // For size_t, uintptr_t
+
+asm(R"(
+.globl _end, __llvm_libc_heap_limit
+.bss
+_end:
+ .fill 8192
+__llvm_libc_heap_limit:
+)");
+
+using LIBC_NAMESPACE::inline_memcpy;
+using LIBC_NAMESPACE::inline_memset;
+using LIBC_NAMESPACE::cpp::byte;
+using LIBC_NAMESPACE::cpp::span;
+using LIBC_NAMESPACE::flat_tlsf::CHUNK_UNIT;
+using LIBC_NAMESPACE::flat_tlsf::flat_tlsf_heap;
+using LIBC_NAMESPACE::flat_tlsf::FlatTlsfHeap;
+using LIBC_NAMESPACE::flat_tlsf::FlatTlsfHeapBuffer;
+
+// We want to test both a locally-instantiated FlatTlsfHeap and the global
+// flat_tlsf_heap pointer.
+#define TEST_FOR_EACH_ALLOCATOR(TestCase, BufferSize) \
+ class LlvmLibcFlatTlsfGlobalTest##TestCase \
+ : public LIBC_NAMESPACE::testing::Test { \
+ public: \
+ FlatTlsfHeapBuffer<BufferSize> fake_global_buffer; \
+ void SetUp() override { \
+ flat_tlsf_heap = \
+ new (&fake_global_buffer) FlatTlsfHeapBuffer<BufferSize>; \
+ } \
+ void RunTest(FlatTlsfHeap &allocator, [[maybe_unused]] size_t N); \
+ }; \
+ TEST_F(LlvmLibcFlatTlsfGlobalTest##TestCase, TestCase) { \
+ alignas(8) byte buf[BufferSize] = {byte(0)}; \
+ FlatTlsfHeap allocator(buf); \
+ RunTest(allocator, BufferSize); \
+ RunTest(*flat_tlsf_heap, flat_tlsf_heap->region().size()); \
+ } \
+ void LlvmLibcFlatTlsfGlobalTest##TestCase::RunTest( \
+ FlatTlsfHeap &allocator, [[maybe_unused]] size_t N)
+
+TEST_FOR_EACH_ALLOCATOR(CanAllocate, 8192) {
+ constexpr size_t ALLOC_SIZE = 256;
+
+ void *ptr = allocator.allocate(ALLOC_SIZE);
+
+ ASSERT_NE(ptr, static_cast<void *>(nullptr));
+}
+
+TEST_FOR_EACH_ALLOCATOR(AllocationsDontOverlap, 8192) {
+ constexpr size_t ALLOC_SIZE = 256;
+
+ void *ptr1 = allocator.allocate(ALLOC_SIZE);
+ void *ptr2 = allocator.allocate(ALLOC_SIZE);
+
+ ASSERT_NE(ptr1, static_cast<void *>(nullptr));
+ ASSERT_NE(ptr2, static_cast<void *>(nullptr));
+
+ uintptr_t ptr1_start = reinterpret_cast<uintptr_t>(ptr1);
+ uintptr_t ptr1_end = ptr1_start + ALLOC_SIZE;
+ uintptr_t ptr2_start = reinterpret_cast<uintptr_t>(ptr2);
+
+ EXPECT_GT(ptr2_start, ptr1_end);
+}
+
+TEST_FOR_EACH_ALLOCATOR(CanFreeAndRealloc, 8192) {
+ constexpr size_t ALLOC_SIZE = 256;
+
+ void *ptr1 = allocator.allocate(ALLOC_SIZE);
+ allocator.free(ptr1);
+ void *ptr2 = allocator.allocate(ALLOC_SIZE);
+
+ EXPECT_EQ(ptr1, ptr2);
+}
+
+TEST_FOR_EACH_ALLOCATOR(ReturnsNullWhenAllocationTooLarge, 8192) {
+ EXPECT_EQ(allocator.allocate(N), static_cast<void *>(nullptr));
+}
+
+TEST(LlvmLibcFlatTlsfGlobal, ReturnsNullWhenFull) {
+ constexpr size_t N = 8192;
+ alignas(8) byte buf[N];
+
+ FlatTlsfHeap allocator(buf);
+
+ bool went_null = false;
+ for (size_t i = 0; i < N; i++) {
+ if (!allocator.allocate(1)) {
+ went_null = true;
+ break;
+ }
+ }
+ EXPECT_TRUE(went_null);
+ EXPECT_EQ(allocator.allocate(1), static_cast<void *>(nullptr));
+}
+
+TEST_FOR_EACH_ALLOCATOR(ReturnedPointersAreAligned, 8192) {
+ void *ptr1 = allocator.allocate(1);
+
+ uintptr_t ptr1_start = reinterpret_cast<uintptr_t>(ptr1);
+ EXPECT_EQ(ptr1_start % CHUNK_UNIT, static_cast<size_t>(0));
+
+ void *ptr2 = allocator.allocate(1);
+ uintptr_t ptr2_start = reinterpret_cast<uintptr_t>(ptr2);
+
+ EXPECT_EQ(ptr2_start % CHUNK_UNIT, static_cast<size_t>(0));
+}
+
+TEST_FOR_EACH_ALLOCATOR(CanRealloc, 8192) {
+ constexpr size_t ALLOC_SIZE = 256;
+ constexpr size_t kNewAllocSize = 512;
+
+ void *ptr1 = allocator.allocate(ALLOC_SIZE);
+ void *ptr2 = allocator.realloc(ptr1, kNewAllocSize);
+
+ ASSERT_NE(ptr1, static_cast<void *>(nullptr));
+ ASSERT_NE(ptr2, static_cast<void *>(nullptr));
+}
+
+TEST_FOR_EACH_ALLOCATOR(ReallocHasSameContent, 8192) {
+ constexpr size_t ALLOC_SIZE = sizeof(int);
+ constexpr size_t kNewAllocSize = sizeof(int) * 2;
+ byte data1[ALLOC_SIZE];
+ byte data2[ALLOC_SIZE];
+
+ int *ptr1 = reinterpret_cast<int *>(allocator.allocate(ALLOC_SIZE));
+ *ptr1 = 42;
+ inline_memcpy(data1, ptr1, ALLOC_SIZE);
+ int *ptr2 = reinterpret_cast<int *>(allocator.realloc(ptr1, kNewAllocSize));
+ inline_memcpy(data2, ptr2, ALLOC_SIZE);
+
+ ASSERT_NE(ptr1, static_cast<int *>(nullptr));
+ ASSERT_NE(ptr2, static_cast<int *>(nullptr));
+
+ // Verify that data inside the allocated and reallocated chunks are the same.
+ bool match = true;
+ for (size_t i = 0; i < ALLOC_SIZE; ++i) {
+ if (data1[i] != data2[i]) {
+ match = false;
+ break;
+ }
+ }
+ EXPECT_TRUE(match);
+}
+
+TEST_FOR_EACH_ALLOCATOR(ReturnsNullReallocFreedPointer, 8192) {
+ constexpr size_t ALLOC_SIZE = 256;
+ constexpr size_t kNewAllocSize = 128;
+
+ void *ptr1 = allocator.allocate(ALLOC_SIZE);
+ allocator.free(ptr1);
+ void *ptr2 = allocator.realloc(ptr1, kNewAllocSize);
+
+ EXPECT_EQ(static_cast<void *>(nullptr), ptr2);
+}
+
+TEST_FOR_EACH_ALLOCATOR(ReallocSmallerSize, 8192) {
+ constexpr size_t ALLOC_SIZE = 256;
+ constexpr size_t kNewAllocSize = 128;
+
+ void *ptr1 = allocator.allocate(ALLOC_SIZE);
+ void *ptr2 = allocator.realloc(ptr1, kNewAllocSize);
+
+ EXPECT_EQ(ptr1, ptr2);
+}
+
+TEST_FOR_EACH_ALLOCATOR(ReallocTooLarge, 8192) {
+ constexpr size_t ALLOC_SIZE = 256;
+ size_t kNewAllocSize = N * 2;
+
+ void *ptr1 = allocator.allocate(ALLOC_SIZE);
+ void *ptr2 = allocator.realloc(ptr1, kNewAllocSize);
+
+ EXPECT_NE(static_cast<void *>(nullptr), ptr1);
+ EXPECT_EQ(static_cast<void *>(nullptr), ptr2);
+}
+
+TEST_FOR_EACH_ALLOCATOR(CanCalloc, 8192) {
+ constexpr size_t ALLOC_SIZE = 64;
+ constexpr size_t NUM = 4;
+ constexpr int size = NUM * ALLOC_SIZE;
+ constexpr byte zero{0};
+
+ byte *ptr1 = reinterpret_cast<byte *>(allocator.calloc(NUM, ALLOC_SIZE));
+
+ for (int i = 0; i < size; i++) {
+ EXPECT_EQ(ptr1[i], zero);
+ }
+}
+
+TEST_FOR_EACH_ALLOCATOR(CanCallocWeirdSize, 8192) {
+ constexpr size_t ALLOC_SIZE = 143;
+ constexpr size_t NUM = 3;
+ constexpr int size = NUM * ALLOC_SIZE;
+ constexpr byte zero{0};
+
+ byte *ptr1 = reinterpret_cast<byte *>(allocator.calloc(NUM, ALLOC_SIZE));
+
+ for (int i = 0; i < size; i++) {
+ EXPECT_EQ(ptr1[i], zero);
+ }
+}
+
+TEST_FOR_EACH_ALLOCATOR(CallocTooLarge, 8192) {
+ size_t ALLOC_SIZE = N + 1;
+ EXPECT_EQ(allocator.calloc(1, ALLOC_SIZE), static_cast<void *>(nullptr));
+}
+
+TEST_FOR_EACH_ALLOCATOR(AllocateZero, 8192) {
+ void *ptr = allocator.allocate(0);
+ ASSERT_EQ(ptr, static_cast<void *>(nullptr));
+}
+
+TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 8192) {
+ constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
+ constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
+
+ for (size_t alignment : ALIGNMENTS) {
+ for (size_t scale : SIZE_SCALES) {
+ size_t size = alignment * scale;
+ void *ptr = allocator.aligned_allocate(alignment, size);
+ EXPECT_NE(ptr, static_cast<void *>(nullptr));
+ EXPECT_EQ(reinterpret_cast<uintptr_t>(ptr) % alignment, size_t(0));
+ allocator.free(ptr);
+ }
+ }
+}
+
+TEST(LlvmLibcFlatTlsfGlobal, AlignedAllocUnalignedBuffer) {
+ alignas(8) byte buf[4096] = {byte(0)};
+
+ FlatTlsfHeap allocator(span<byte>(buf).subspan(1));
+
+ constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
+ constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
+
+ for (size_t alignment : ALIGNMENTS) {
+ for (size_t scale : SIZE_SCALES) {
+ size_t size = alignment * scale;
+ void *ptr = allocator.aligned_allocate(alignment, size);
+ EXPECT_NE(ptr, static_cast<void *>(nullptr));
+ EXPECT_EQ(reinterpret_cast<uintptr_t>(ptr) % alignment, size_t(0));
+ allocator.free(ptr);
+ }
+ }
+}
+
+TEST_FOR_EACH_ALLOCATOR(InvalidAlignedAllocAlignment, 8192) {
+ constexpr size_t ALIGNMENTS[] = {4, 8, 16, 32, 64, 128, 256};
+ for (size_t alignment : ALIGNMENTS) {
+ void *ptr = allocator.aligned_allocate(alignment - 1, alignment - 1);
+ EXPECT_EQ(ptr, static_cast<void *>(nullptr));
+ }
+
+ for (size_t alignment : ALIGNMENTS) {
+ void *ptr = allocator.aligned_allocate(alignment, alignment + 1);
+ EXPECT_EQ(ptr, static_cast<void *>(nullptr));
+ }
+
+ void *ptr = allocator.aligned_allocate(1, 0);
+ EXPECT_EQ(ptr, static_cast<void *>(nullptr));
+
+ ptr = allocator.aligned_allocate(0, 8);
+ EXPECT_EQ(ptr, static_cast<void *>(nullptr));
+}
diff --git a/libc/test/src/__support/flat_tlsf/heap_test.cpp b/libc/test/src/__support/flat_tlsf/heap_test.cpp
new file mode 100644
index 0000000000000..fc789281e5476
--- /dev/null
+++ b/libc/test/src/__support/flat_tlsf/heap_test.cpp
@@ -0,0 +1,219 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for flat_tlsf Heap.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/algorithm.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/flat_tlsf/binning.h"
+#include "src/__support/flat_tlsf/chunk.h"
+#include "src/__support/flat_tlsf/common.h"
+#include "src/__support/flat_tlsf/heap.h"
+#include "src/__support/flat_tlsf/node.h"
+#include "src/__support/flat_tlsf/tag.h"
+#include "src/string/memory_utils/inline_memset.h"
+#include "test/UnitTest/Test.h"
+#include <stddef.h> // For max_align_t, size_t
+
+using LIBC_NAMESPACE::cpp::max;
+using LIBC_NAMESPACE::cpp::min;
+using LIBC_NAMESPACE::cpp::numeric_limits;
+using LIBC_NAMESPACE::flat_tlsf::Binning;
+using LIBC_NAMESPACE::flat_tlsf::Byte;
+using LIBC_NAMESPACE::flat_tlsf::CHUNK_UNIT;
+using LIBC_NAMESPACE::flat_tlsf::GAP_BIN_OFFSET;
+using LIBC_NAMESPACE::flat_tlsf::GAP_HIGH_SIZE_OFFSET;
+using LIBC_NAMESPACE::flat_tlsf::GAP_LOW_SIZE_OFFSET;
+using LIBC_NAMESPACE::flat_tlsf::GAP_NODE_OFFSET;
+using LIBC_NAMESPACE::flat_tlsf::Heap;
+using LIBC_NAMESPACE::flat_tlsf::Node;
+namespace chunk = LIBC_NAMESPACE::flat_tlsf::chunk;
+namespace tag = LIBC_NAMESPACE::flat_tlsf::tag;
+using LIBC_NAMESPACE::inline_memset;
+
+namespace {
+
+struct Layout {
+ size_t size;
+ size_t align;
+};
+
+LIBC_INLINE constexpr size_t min_first_heap_size() {
+ size_t size = chunk::required_chunk_size(Binning::BIN_COUNT * sizeof(Node *));
+ size_t max_overhead = CHUNK_UNIT + alignof(size_t) - 1;
+ return size + max_overhead;
+}
+
+LIBC_INLINE constexpr Layout min_first_heap_layout() {
+ size_t size = Binning::BIN_COUNT * sizeof(size_t);
+ size_t max_overhead = CHUNK_UNIT;
+ return {size + max_overhead, alignof(size_t)};
+}
+
+TEST(LlvmLibcFlatTlsfHeapTest, VerifyGapProperties) {
+ Heap heap;
+
+ constexpr Layout meta_layout = min_first_heap_layout();
+ alignas(meta_layout.align) Byte meta_mem[meta_layout.size] = {};
+ Byte *meta_heap_end = heap.claim(meta_mem, meta_layout.size);
+ ASSERT_NE(meta_heap_end, static_cast<Byte *>(nullptr));
+
+ Byte gap_mem[999] = {};
+ Byte *gap_end = heap.claim(gap_mem, 999);
+ ASSERT_NE(gap_end, static_cast<Byte *>(nullptr));
+
+ ASSERT_LE(gap_end, gap_mem + 999);
+ ASSERT_GT(gap_end + CHUNK_UNIT, gap_mem + 999);
+
+ Byte *gap_base = chunk::align_up(gap_mem + sizeof(Byte));
+ size_t gap_size = static_cast<size_t>(gap_end - gap_base);
+ ASSERT_LE(gap_size, size_t{999});
+ ASSERT_GT(gap_size, size_t{999} - CHUNK_UNIT * 2);
+
+ uint32_t gap_bin = min(Binning::size_to_bin(gap_size),
+ static_cast<uint32_t>(Binning::BIN_COUNT - 1));
+
+ Node *gap_node_ptr = heap.get_gap_list_head(gap_bin);
+ ASSERT_NE(gap_node_ptr, static_cast<Node *>(nullptr));
+ ASSERT_EQ(gap_node_ptr, chunk::gap_base_to_node(gap_base));
+
+ Node gap_node = *gap_node_ptr;
+ ASSERT_EQ(gap_node.next, static_cast<Node *>(nullptr));
+ ASSERT_EQ(gap_node.next_of_prev, heap.get_gap_list_ptr(gap_bin));
+
+ ASSERT_EQ(gap_bin,
+ chunk::read_word<uint32_t>(chunk::gap_base_to_bin(gap_base)));
+ ASSERT_EQ(gap_size,
+ chunk::read_word<size_t>(chunk::gap_base_to_size(gap_base)));
+
+ ASSERT_EQ(chunk::read_word<size_t>(chunk::gap_base_to_size(gap_base)),
+ gap_size);
+ ASSERT_EQ(chunk::gap_end_to_size_and_flag(gap_end),
+ reinterpret_cast<size_t *>(gap_end - sizeof(size_t)));
+ ASSERT_EQ(chunk::read_word<size_t>(chunk::gap_end_to_size_and_flag(gap_end)),
+ gap_size);
+
+ heap.test_deregister_gap(gap_base, gap_size);
+}
+
+TEST(LlvmLibcFlatTlsfHeapTest, AllocDeallocTest) {
+ Byte arena[5000] = {};
+ Heap heap;
+ ASSERT_NE(heap.claim(arena, 5000), static_cast<Byte *>(nullptr));
+
+ size_t size = 2435;
+ size_t align = 8;
+ Byte *allocation = heap.allocate(size, align);
+ ASSERT_NE(allocation, static_cast<Byte *>(nullptr));
+
+ for (size_t i = 0; i < size; ++i) {
+ allocation[i] = static_cast<Byte>(0xCD);
+ }
+
+ heap.deallocate(allocation, size);
+}
+
+TEST(LlvmLibcFlatTlsfHeapTest, AllocFailTest) {
+ constexpr size_t arena_size = min_first_heap_size() + 100 + CHUNK_UNIT;
+ Byte arena[arena_size] = {};
+ Heap heap;
+ ASSERT_NE(heap.claim(arena, arena_size), static_cast<Byte *>(nullptr));
+
+ Byte *a1 = heap.allocate(8, 8);
+ ASSERT_NE(a1, static_cast<Byte *>(nullptr));
+
+ size_t large_size = 1234 + CHUNK_UNIT;
+ Byte *a2 = heap.allocate(large_size, 8);
+ ASSERT_EQ(a2, static_cast<Byte *>(nullptr));
+}
+
+TEST(LlvmLibcFlatTlsfHeapTest, ClaimHeapThatsTooSmall) {
+ alignas(8) Byte tiny_heap[200] = {};
+ Heap heap;
+ ASSERT_EQ(heap.claim(tiny_heap, 200), static_cast<Byte *>(nullptr));
+
+ ASSERT_EQ(heap.get_gap_list(), static_cast<Node **>(nullptr));
+ ASSERT_GE(heap.get_available().bit_scan_after(0),
+ static_cast<uint32_t>(Binning::BIN_COUNT));
+}
+
+TEST(LlvmLibcFlatTlsfHeapTest, ClaimSmallHeapAfterMetadataIsAllocated) {
+ constexpr Layout meta_layout = min_first_heap_layout();
+ alignas(meta_layout.align) Byte big_heap[meta_layout.size] = {};
+
+ Heap heap;
+ ASSERT_NE(heap.claim(big_heap, meta_layout.size),
+ static_cast<Byte *>(nullptr));
+
+ ASSERT_NE(heap.get_gap_list(), static_cast<Node **>(nullptr));
+ ASSERT_GE(heap.get_available().bit_scan_after(0),
+ static_cast<uint32_t>(Binning::BIN_COUNT));
+
+ alignas(8) Byte tiny_heap[300] = {};
+ ASSERT_NE(heap.claim(tiny_heap, 300), static_cast<Byte *>(nullptr));
+}
+
+TEST(LlvmLibcFlatTlsfHeapTest, MallocFreeBasic) {
+ Byte arena[5000] = {};
+ Heap heap;
+ ASSERT_NE(heap.claim(arena, 5000), static_cast<Byte *>(nullptr));
+
+ // Test simple malloc/free
+ void *p1 = heap.malloc(100);
+ ASSERT_NE(p1, static_cast<void *>(nullptr));
+ ASSERT_EQ(reinterpret_cast<uintptr_t>(p1) % alignof(max_align_t), size_t{0});
+
+ inline_memset(p1, 0xAB, 100);
+ heap.free(p1);
+
+ // Test aligned_alloc
+ void *p2 = heap.aligned_alloc(64, 100);
+ ASSERT_NE(p2, static_cast<void *>(nullptr));
+ ASSERT_EQ(reinterpret_cast<uintptr_t>(p2) % 64, size_t{0});
+ inline_memset(p2, 0xBC, 100);
+ heap.free(p2);
+
+ // Test free(nullptr) is a safe no-op
+ heap.free(nullptr);
+}
+
+TEST(LlvmLibcFlatTlsfHeapTest, ReallocBasic) {
+ Byte arena[5000] = {};
+ Heap heap;
+ ASSERT_NE(heap.claim(arena, 5000), static_cast<Byte *>(nullptr));
+
+ // realloc(nullptr, size) is equivalent to malloc
+ void *p1 = heap.realloc(nullptr, 100);
+ ASSERT_NE(p1, static_cast<void *>(nullptr));
+ inline_memset(p1, 0x11, 100);
+
+ // Grow in place (or via move)
+ void *p2 = heap.realloc(p1, 200);
+ ASSERT_NE(p2, static_cast<void *>(nullptr));
+ Byte *p2_bytes = static_cast<Byte *>(p2);
+ for (size_t i = 0; i < 100; ++i) {
+ ASSERT_EQ(p2_bytes[i], static_cast<Byte>(0x11));
+ }
+
+ // Shrink in place
+ void *p3 = heap.realloc(p2, 50);
+ ASSERT_EQ(p3, p2); // Shrink should always be in-place
+ Byte *p3_bytes = static_cast<Byte *>(p3);
+ for (size_t i = 0; i < 50; ++i) {
+ ASSERT_EQ(p3_bytes[i], static_cast<Byte>(0x11));
+ }
+
+ // realloc(ptr, 0) is equivalent to free
+ void *p4 = heap.realloc(p3, 0);
+ ASSERT_EQ(p4, static_cast<void *>(nullptr));
+}
+
+} // namespace
diff --git a/libc/test/src/__support/flat_tlsf/node_test.cpp b/libc/test/src/__support/flat_tlsf/node_test.cpp
new file mode 100644
index 0000000000000..661c72b60cf48
--- /dev/null
+++ b/libc/test/src/__support/flat_tlsf/node_test.cpp
@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for flat_tlsf Node.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/flat_tlsf/node.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::flat_tlsf::Node;
+
+TEST(LlvmLibcFlatTlsfNodeTest, BasicOperations) {
+ Node x{nullptr, nullptr};
+ Node y{nullptr, nullptr};
+ Node z{nullptr, nullptr};
+
+ y.link_at(Node{nullptr, x.addr_of_next()});
+ z.link_at(Node{&y, x.addr_of_next()});
+
+ {
+ Node *curr = &x;
+ ASSERT_NE(curr, static_cast<Node *>(nullptr));
+ EXPECT_EQ(curr, &x);
+ curr = curr->next;
+ ASSERT_NE(curr, static_cast<Node *>(nullptr));
+ EXPECT_EQ(curr, &z);
+ curr = curr->next;
+ ASSERT_NE(curr, static_cast<Node *>(nullptr));
+ EXPECT_EQ(curr, &y);
+ curr = curr->next;
+ EXPECT_EQ(curr, static_cast<Node *>(nullptr));
+ }
+
+ {
+ Node *curr = &y;
+ ASSERT_NE(curr, static_cast<Node *>(nullptr));
+ EXPECT_EQ(curr, &y);
+ curr = curr->next;
+ EXPECT_EQ(curr, static_cast<Node *>(nullptr));
+ }
+
+ z.unlink();
+
+ {
+ Node *curr = &x;
+ ASSERT_NE(curr, static_cast<Node *>(nullptr));
+ EXPECT_EQ(curr, &x);
+ curr = curr->next;
+ ASSERT_NE(curr, static_cast<Node *>(nullptr));
+ EXPECT_EQ(curr, &y);
+ curr = curr->next;
+ EXPECT_EQ(curr, static_cast<Node *>(nullptr));
+ }
+
+ z.link_at(Node{&y, x.addr_of_next()});
+
+ {
+ Node *curr = &x;
+ ASSERT_NE(curr, static_cast<Node *>(nullptr));
+ EXPECT_EQ(curr, &x);
+ curr = curr->next;
+ ASSERT_NE(curr, static_cast<Node *>(nullptr));
+ EXPECT_EQ(curr, &z);
+ curr = curr->next;
+ ASSERT_NE(curr, static_cast<Node *>(nullptr));
+ EXPECT_EQ(curr, &y);
+ curr = curr->next;
+ EXPECT_EQ(curr, static_cast<Node *>(nullptr));
+ }
+
+ z.unlink();
+ y.unlink();
+
+ {
+ Node *curr = &x;
+ ASSERT_NE(curr, static_cast<Node *>(nullptr));
+ EXPECT_EQ(curr, &x);
+ curr = curr->next;
+ EXPECT_EQ(curr, static_cast<Node *>(nullptr));
+ }
+}
diff --git a/libc/test/src/__support/flat_tlsf/tag_test.cpp b/libc/test/src/__support/flat_tlsf/tag_test.cpp
new file mode 100644
index 0000000000000..88eb07f97db76
--- /dev/null
+++ b/libc/test/src/__support/flat_tlsf/tag_test.cpp
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for flat_tlsf tag utilities.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/flat_tlsf/common.h"
+#include "src/__support/flat_tlsf/tag.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::flat_tlsf::Byte;
+namespace tag = LIBC_NAMESPACE::flat_tlsf::tag;
+
+TEST(LlvmLibcFlatTlsfTagTest, ReadsFlags) {
+ EXPECT_FALSE(tag::is_above_free(0));
+ EXPECT_FALSE(tag::is_allocated(0));
+ EXPECT_FALSE(tag::is_heap_base(0));
+ EXPECT_FALSE(tag::is_heap_end(0));
+
+ EXPECT_TRUE(tag::is_above_free(tag::ABOVE_FREE_FLAG));
+ EXPECT_TRUE(tag::is_allocated(tag::ALLOCATED_FLAG));
+ EXPECT_TRUE(tag::is_heap_base(tag::HEAP_BASE_FLAG));
+ EXPECT_TRUE(tag::is_heap_end(tag::HEAP_END_FLAG));
+
+ Byte all_flags = tag::ABOVE_FREE_FLAG | tag::ALLOCATED_FLAG |
+ tag::HEAP_BASE_FLAG | tag::HEAP_END_FLAG;
+ EXPECT_TRUE(tag::is_above_free(all_flags));
+ EXPECT_TRUE(tag::is_allocated(all_flags));
+ EXPECT_TRUE(tag::is_heap_base(all_flags));
+ EXPECT_TRUE(tag::is_heap_end(all_flags));
+}
+
+TEST(LlvmLibcFlatTlsfTagTest, MutatesFlags) {
+ Byte byte = tag::ALLOCATED_FLAG;
+
+ tag::set_above_free(&byte);
+ EXPECT_TRUE(tag::is_above_free(byte));
+ EXPECT_TRUE(tag::is_allocated(byte));
+
+ tag::clear_above_free(&byte);
+ EXPECT_FALSE(tag::is_above_free(byte));
+ EXPECT_TRUE(tag::is_allocated(byte));
+
+ tag::set_end_flag(&byte);
+ EXPECT_TRUE(tag::is_heap_end(byte));
+ EXPECT_TRUE(tag::is_allocated(byte));
+
+ tag::clear_end_flag(&byte);
+ EXPECT_FALSE(tag::is_heap_end(byte));
+ EXPECT_TRUE(tag::is_allocated(byte));
+}
>From 89190d0297ee09c7e6b6c7560e1ef4906b780f9a Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 27 May 2026 16:10:28 -0700
Subject: [PATCH 2/8] remove goto
---
libc/src/__support/flat_tlsf/heap.h | 47 +++++++++++++++--------------
1 file changed, 25 insertions(+), 22 deletions(-)
diff --git a/libc/src/__support/flat_tlsf/heap.h b/libc/src/__support/flat_tlsf/heap.h
index 3e9dbffc4c9c6..ebcc0a6cc4f0a 100644
--- a/libc/src/__support/flat_tlsf/heap.h
+++ b/libc/src/__support/flat_tlsf/heap.h
@@ -167,24 +167,23 @@ class Heap {
size_t required_chunk_size = chunk::required_chunk_size(required_size);
Byte *base = nullptr;
Byte *chunk_end = nullptr;
+ do {
+ size_t bin = Binning::size_to_bin_ceil(
+ cpp::max(required_chunk_size, required_align));
- size_t bin = Binning::size_to_bin_ceil(
- cpp::max(required_chunk_size, required_align));
-
- if (bin >= Binning::BIN_COUNT - 1) {
- if (available.read_bit(Binning::BIN_COUNT - 1)) {
- if (auto result =
- full_search_bin(Binning::BIN_COUNT - 1, required_chunk_size,
- required_align - 1)) {
- base = result->base;
- chunk_end = result->end;
- goto success;
+ if (bin >= Binning::BIN_COUNT - 1) {
+ if (available.read_bit(Binning::BIN_COUNT - 1)) {
+ if (auto result =
+ full_search_bin(Binning::BIN_COUNT - 1, required_chunk_size,
+ required_align - 1)) {
+ base = result->base;
+ chunk_end = result->end;
+ break;
+ }
}
+ return nullptr;
}
- return nullptr;
- }
- {
size_t bit = available.bit_scan_after(static_cast<uint32_t>(bin));
if (bit >= Binning::BIN_COUNT) {
if (available.read_bit(static_cast<uint32_t>(bin - 1))) {
@@ -193,7 +192,7 @@ class Heap {
required_chunk_size, required_align - 1)) {
base = result->base;
chunk_end = result->end;
- goto success;
+ break;
}
}
return nullptr;
@@ -209,9 +208,10 @@ class Heap {
deregister_gap(base, size);
tag::clear_above_free(chunk::end_to_tag(base));
chunk_end = base + size;
- goto success;
+ break;
} else {
size_t align_mask = required_align - 1;
+ bool success = false;
while (true) {
for (Node *node = gap_list[bit]; node != nullptr; node = node->next) {
size_t size =
@@ -228,9 +228,12 @@ class Heap {
}
base = aligned_base;
chunk_end = end;
- goto success;
+ success = true;
+ break;
}
}
+ if (success)
+ break;
if (bit + 1 < Binning::BIN_COUNT ||
BitField::BITS > Binning::BIN_COUNT) {
@@ -256,16 +259,16 @@ class Heap {
base = aligned_base;
chunk_end = end;
- goto success;
+ success = true;
+ break;
}
}
-
+ if (success)
+ break;
return nullptr;
}
}
- }
-
- success:
+ } while (0);
LIBC_ASSERT(chunk::align_down(base) == base);
Byte *end = base + required_chunk_size;
>From 18bd0870192284df1a49d59940d5665e668c8a56 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 27 May 2026 16:18:43 -0700
Subject: [PATCH 3/8] fix
---
libc/src/__support/flat_tlsf/global.h | 4 ++--
libc/src/__support/flat_tlsf/heap.h | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/libc/src/__support/flat_tlsf/global.h b/libc/src/__support/flat_tlsf/global.h
index d14fec35d318d..0370326915672 100644
--- a/libc/src/__support/flat_tlsf/global.h
+++ b/libc/src/__support/flat_tlsf/global.h
@@ -89,8 +89,8 @@ class FlatTlsfHeap {
private:
LIBC_INLINE void init() {
LIBC_ASSERT(heap.get_gap_list() == nullptr && "duplicate initialization");
- Byte *res = heap.claim(reinterpret_cast<Byte *>(begin),
- static_cast<size_t>(end - begin));
+ [[maybe_unused]] Byte *res = heap.claim(reinterpret_cast<Byte *>(begin),
+ static_cast<size_t>(end - begin));
LIBC_ASSERT(res != nullptr && "Failed to claim memory for heap");
}
diff --git a/libc/src/__support/flat_tlsf/heap.h b/libc/src/__support/flat_tlsf/heap.h
index ebcc0a6cc4f0a..814f54b1e2c10 100644
--- a/libc/src/__support/flat_tlsf/heap.h
+++ b/libc/src/__support/flat_tlsf/heap.h
@@ -84,7 +84,7 @@ class Heap {
LIBC_ASSERT(*bin_ptr != nullptr);
}
- LIBC_INLINE void deregister_gap(Byte *base, size_t size) {
+ LIBC_INLINE void deregister_gap(Byte *base, [[maybe_unused]] size_t size) {
LIBC_ASSERT(
gap_list[cpp::min(Binning::size_to_bin(size),
static_cast<uint32_t>(Binning::BIN_COUNT - 1))] !=
>From 4c5633644333f21515de15730a8fd02e44fd2006 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 27 May 2026 16:26:27 -0700
Subject: [PATCH 4/8] fix
---
libc/src/__support/flat_tlsf/bitfield.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/src/__support/flat_tlsf/bitfield.h b/libc/src/__support/flat_tlsf/bitfield.h
index 93b4dc895492c..224b137d244ec 100644
--- a/libc/src/__support/flat_tlsf/bitfield.h
+++ b/libc/src/__support/flat_tlsf/bitfield.h
@@ -24,7 +24,7 @@ namespace LIBC_NAMESPACE_DECL {
namespace flat_tlsf {
struct alignas(16) BitField {
- static constexpr size_t BITS_PER_ELEMENT = CHAR_WIDTH * sizeof(size_t);
+ static constexpr size_t BITS_PER_ELEMENT = 8 * sizeof(size_t);
static constexpr size_t NUMBER_OF_ELEMENTS = 3;
static constexpr size_t BITS = BITS_PER_ELEMENT * NUMBER_OF_ELEMENTS;
>From 8a73fed2cb797932a360c01e92c214a0ac39d7d0 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Wed, 27 May 2026 16:27:10 -0700
Subject: [PATCH 5/8] fix
---
libc/src/__support/flat_tlsf/bitfield.h | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/libc/src/__support/flat_tlsf/bitfield.h b/libc/src/__support/flat_tlsf/bitfield.h
index 224b137d244ec..7a8cdf0494784 100644
--- a/libc/src/__support/flat_tlsf/bitfield.h
+++ b/libc/src/__support/flat_tlsf/bitfield.h
@@ -16,6 +16,7 @@
#include "hdr/stdint_proxy.h"
#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/limits.h"
#include "src/__support/flat_tlsf/bit_utils.h"
#include "src/__support/macros/attributes.h"
#include "src/__support/macros/config.h"
@@ -24,7 +25,8 @@ namespace LIBC_NAMESPACE_DECL {
namespace flat_tlsf {
struct alignas(16) BitField {
- static constexpr size_t BITS_PER_ELEMENT = 8 * sizeof(size_t);
+ static constexpr size_t BITS_PER_ELEMENT =
+ cpp::numeric_limits<size_t>::digits;
static constexpr size_t NUMBER_OF_ELEMENTS = 3;
static constexpr size_t BITS = BITS_PER_ELEMENT * NUMBER_OF_ELEMENTS;
>From 0836583eceaa4850d0f4037fe645fa946f807bd8 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Thu, 28 May 2026 09:09:25 -0700
Subject: [PATCH 6/8] fix bugs and add a fuzzer
---
libc/fuzzing/__support/CMakeLists.txt | 14 ++
libc/fuzzing/__support/tlsf_heap_fuzz.cpp | 221 ++++++++++++++++++
libc/src/__support/flat_tlsf/binning.h | 6 +-
libc/src/__support/flat_tlsf/heap.h | 29 ++-
libc/src/__support/flat_tlsf/tag.h | 6 +-
.../src/__support/flat_tlsf/binning_test.cpp | 2 +-
.../test/src/__support/flat_tlsf/tag_test.cpp | 20 ++
7 files changed, 280 insertions(+), 18 deletions(-)
create mode 100644 libc/fuzzing/__support/tlsf_heap_fuzz.cpp
diff --git a/libc/fuzzing/__support/CMakeLists.txt b/libc/fuzzing/__support/CMakeLists.txt
index be72259036458..569b26bb4d9d2 100644
--- a/libc/fuzzing/__support/CMakeLists.txt
+++ b/libc/fuzzing/__support/CMakeLists.txt
@@ -52,4 +52,18 @@ if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_GPU)
PROPERTIES
MSVC_DEBUG_INFORMATION_FORMAT ""
)
+
+ add_libc_fuzzer(
+ tlsf_heap_fuzz
+ SRCS
+ tlsf_heap_fuzz.cpp
+ DEPENDS
+ libc.src.__support.flat_tlsf.heap
+ )
+ get_fq_target_name(tlsf_heap_fuzz tlsf_heap_fuzz_target_name)
+ set_target_properties(
+ ${tlsf_heap_fuzz_target_name}
+ PROPERTIES
+ MSVC_DEBUG_INFORMATION_FORMAT ""
+ )
endif()
diff --git a/libc/fuzzing/__support/tlsf_heap_fuzz.cpp b/libc/fuzzing/__support/tlsf_heap_fuzz.cpp
new file mode 100644
index 0000000000000..d9c3b16ddbf49
--- /dev/null
+++ b/libc/fuzzing/__support/tlsf_heap_fuzz.cpp
@@ -0,0 +1,221 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// Fuzzing test for llvm-libc TLSF-based heap implementation.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/bit.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/flat_tlsf/heap.h"
+#include "src/string/memory_utils/inline_memcpy.h"
+#include "src/string/memory_utils/inline_memmove.h"
+#include "src/string/memory_utils/inline_memset.h"
+#include <stddef.h> // For max_align_t
+
+using LIBC_NAMESPACE::inline_memcpy;
+using LIBC_NAMESPACE::inline_memmove;
+using LIBC_NAMESPACE::inline_memset;
+using LIBC_NAMESPACE::cpp::nullopt;
+using LIBC_NAMESPACE::cpp::optional;
+using LIBC_NAMESPACE::flat_tlsf::Byte;
+using LIBC_NAMESPACE::flat_tlsf::Heap;
+
+// Record of an outstanding allocation.
+struct Alloc {
+ void *ptr;
+ size_t size;
+ size_t alignment;
+ uint8_t canary; // Byte written to the allocation
+};
+
+// A simple vector that tracks allocations using the heap.
+class AllocVec {
+public:
+ AllocVec(Heap &heap) : heap(&heap), size_(0), capacity(0) {
+ allocs = nullptr;
+ }
+
+ bool empty() const { return !size_; }
+
+ size_t size() const { return size_; }
+
+ bool push_back(Alloc alloc) {
+ if (size_ == capacity) {
+ size_t new_cap = capacity ? capacity * 2 : 1;
+ Alloc *new_allocs = reinterpret_cast<Alloc *>(
+ heap->realloc(allocs, new_cap * sizeof(Alloc)));
+ if (!new_allocs)
+ return false;
+ allocs = new_allocs;
+ capacity = new_cap;
+ }
+ allocs[size_++] = alloc;
+ return true;
+ }
+
+ Alloc &operator[](size_t idx) { return allocs[idx]; }
+
+ void erase_idx(size_t idx) {
+ inline_memmove(&allocs[idx], &allocs[idx + 1],
+ sizeof(Alloc) * (size_ - idx - 1));
+ --size_;
+ }
+
+private:
+ Heap *heap;
+ Alloc *allocs;
+ size_t size_;
+ size_t capacity;
+};
+
+// Choose a T value by casting libfuzzer data or exit.
+template <typename T>
+optional<T> choose(const uint8_t *&data, size_t &remainder) {
+ if (sizeof(T) > remainder)
+ return nullopt;
+ T out;
+ inline_memcpy(&out, data, sizeof(T));
+ data += sizeof(T);
+ remainder -= sizeof(T);
+ return out;
+}
+
+// The type of allocation to perform
+enum class AllocType : uint8_t {
+ MALLOC,
+ ALIGNED_ALLOC,
+ REALLOC,
+ CALLOC,
+ NUM_ALLOC_TYPES,
+};
+
+template <>
+optional<AllocType> choose<AllocType>(const uint8_t *&data, size_t &remainder) {
+ auto raw = choose<uint8_t>(data, remainder);
+ if (!raw)
+ return nullopt;
+ return static_cast<AllocType>(
+ *raw % static_cast<uint8_t>(AllocType::NUM_ALLOC_TYPES));
+}
+
+constexpr size_t heap_size = 64 * 1024;
+
+optional<size_t> choose_size(const uint8_t *&data, size_t &remainder) {
+ auto raw = choose<size_t>(data, remainder);
+ if (!raw)
+ return nullopt;
+ return *raw % heap_size;
+}
+
+optional<size_t> choose_alloc_idx(const AllocVec &allocs, const uint8_t *&data,
+ size_t &remainder) {
+ if (allocs.empty())
+ return nullopt;
+ auto raw = choose<size_t>(data, remainder);
+ if (!raw)
+ return nullopt;
+ return *raw % allocs.size();
+}
+
+#define ASSIGN_OR_RETURN(TYPE, NAME, EXPR) \
+ auto maybe_##NAME = EXPR; \
+ if (!maybe_##NAME) \
+ return 0; \
+ TYPE NAME = *maybe_##NAME
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) {
+ alignas(64) Byte arena[heap_size] = {};
+ Heap heap;
+ if (heap.claim(arena, heap_size) == nullptr)
+ return 0;
+
+ AllocVec allocs(heap);
+
+ uint8_t canary = 0;
+ while (true) {
+ ASSIGN_OR_RETURN(auto, should_alloc, choose<bool>(data, remainder));
+ if (should_alloc) {
+ ASSIGN_OR_RETURN(auto, alloc_type, choose<AllocType>(data, remainder));
+ ASSIGN_OR_RETURN(size_t, alloc_size, choose_size(data, remainder));
+
+ // Perform allocation.
+ void *ptr = nullptr;
+ size_t alignment = alignof(max_align_t);
+ switch (alloc_type) {
+ case AllocType::MALLOC:
+ ptr = heap.malloc(alloc_size);
+ break;
+ case AllocType::ALIGNED_ALLOC: {
+ ASSIGN_OR_RETURN(size_t, chosen_align, choose_size(data, remainder));
+ alignment = LIBC_NAMESPACE::cpp::bit_ceil(chosen_align);
+ if (alignment < sizeof(void *))
+ alignment = sizeof(void *);
+ ptr = heap.aligned_alloc(alignment, alloc_size);
+ break;
+ }
+ case AllocType::REALLOC: {
+ if (!alloc_size)
+ return 0;
+ ASSIGN_OR_RETURN(size_t, idx,
+ choose_alloc_idx(allocs, data, remainder));
+ Alloc &alloc = allocs[idx];
+ ptr = heap.realloc(alloc.ptr, alloc_size);
+ if (ptr) {
+ // Extend the canary region if necessary.
+ if (alloc_size > alloc.size)
+ inline_memset(static_cast<char *>(ptr) + alloc.size, alloc.canary,
+ alloc_size - alloc.size);
+ alloc.ptr = ptr;
+ alloc.size = alloc_size;
+ }
+ break;
+ }
+ case AllocType::CALLOC: {
+ ptr = heap.malloc(alloc_size);
+ if (ptr) {
+ inline_memset(ptr, 0, alloc_size);
+ }
+ break;
+ }
+ default:
+ __builtin_trap();
+ }
+
+ if (ptr) {
+ if (alloc_type != AllocType::REALLOC) {
+ // Fill with a new canary and track the allocation.
+ inline_memset(ptr, canary, alloc_size);
+ Alloc alloc = {ptr, alloc_size, alignment, canary};
+ if (!allocs.push_back(alloc))
+ heap.free(ptr);
+ canary++;
+ }
+ }
+ } else {
+ // Select a random allocation.
+ ASSIGN_OR_RETURN(size_t, idx, choose_alloc_idx(allocs, data, remainder));
+ Alloc &alloc = allocs[idx];
+
+ // Check alignment.
+ if (reinterpret_cast<uintptr_t>(alloc.ptr) % alloc.alignment)
+ __builtin_trap();
+
+ // Check the canary.
+ uint8_t *ptr = reinterpret_cast<uint8_t *>(alloc.ptr);
+ for (size_t i = 0; i < alloc.size; ++i)
+ if (ptr[i] != alloc.canary)
+ __builtin_trap();
+
+ // Free the allocation and untrack it.
+ heap.free(alloc.ptr);
+ allocs.erase_idx(idx);
+ }
+ }
+ return 0;
+}
diff --git a/libc/src/__support/flat_tlsf/binning.h b/libc/src/__support/flat_tlsf/binning.h
index 6745b19c201ac..17e390085d836 100644
--- a/libc/src/__support/flat_tlsf/binning.h
+++ b/libc/src/__support/flat_tlsf/binning.h
@@ -75,7 +75,7 @@ struct Binning {
/// performance.
template <size_t LIN_DIVS, size_t LIN_EXT_MULTI>
LIBC_INLINE static constexpr size_t
- linear_extend_then_linearly_divided_expotential_binning(size_t size) {
+ linear_extend_then_linearly_divided_exponential_binning(size_t size) {
static_assert(bit_utils::is_power_of_2(LIN_DIVS),
"LIN_DIVS must be a power of two");
static_assert(bit_utils::is_power_of_2(LIN_EXT_MULTI),
@@ -144,10 +144,10 @@ struct Binning {
LIBC_INLINE constexpr static uint32_t size_to_bin(size_t size) {
if constexpr (sizeof(void *) == 8)
return static_cast<uint32_t>(
- linear_extend_then_linearly_divided_expotential_binning<8, 4>(size));
+ linear_extend_then_linearly_divided_exponential_binning<8, 4>(size));
else
return static_cast<uint32_t>(
- linear_extend_then_linearly_divided_expotential_binning<4, 4>(size));
+ linear_extend_then_linearly_divided_exponential_binning<4, 4>(size));
}
LIBC_INLINE constexpr static uint32_t size_to_bin_ceil(size_t size) {
diff --git a/libc/src/__support/flat_tlsf/heap.h b/libc/src/__support/flat_tlsf/heap.h
index 814f54b1e2c10..6479767c1cf90 100644
--- a/libc/src/__support/flat_tlsf/heap.h
+++ b/libc/src/__support/flat_tlsf/heap.h
@@ -16,6 +16,7 @@
#include "hdr/types/size_t.h"
#include "src/__support/CPP/algorithm.h"
+#include "src/__support/CPP/bit.h"
#include "src/__support/CPP/limits.h"
#include "src/__support/CPP/optional.h"
#include "src/__support/flat_tlsf/binning.h"
@@ -111,7 +112,8 @@ class Heap {
// Gap lists haven't been initialized
if (gap_list == nullptr) {
- base = cpp::max(cpp::bit_cast<Byte *>(uintptr_t{1}), base);
+ if (base == nullptr)
+ base = cpp::bit_cast<Byte *>(uintptr_t{1});
heap_base = bit_utils::align_up_by(base, alignof(Node *));
size_t gap_list_size = sizeof(Node *) * Binning::BIN_COUNT;
gap_base = chunk::align_up(heap_base + gap_list_size + sizeof(Byte));
@@ -406,6 +408,8 @@ class Heap {
}
static constexpr size_t HEADER_SIZE = sizeof(size_t);
+ static constexpr size_t SHIFT_MASK = CHUNK_UNIT - 1;
+ static constexpr size_t MIN_SHIFT_EXPONENT = cpp::countr_zero(HEADER_SIZE);
LIBC_INLINE void *malloc(size_t size) {
return aligned_alloc(CHUNK_UNIT, size);
@@ -419,6 +423,11 @@ class Heap {
size_t allocated_align = cpp::max(align, header_align);
size_t shift = (HEADER_SIZE + align - 1) & ~(align - 1);
+ size_t shift_exponent = static_cast<size_t>(cpp::countr_zero(shift));
+ LIBC_ASSERT(shift_exponent <= SHIFT_MASK && "Alignment too large for allocation header");
+ if (shift_exponent > SHIFT_MASK)
+ return nullptr;
+
size_t allocated_size = size + shift;
Byte *base_ptr = allocate(allocated_size, allocated_align);
@@ -428,10 +437,8 @@ class Heap {
size_t actual_chunk_size = chunk::required_chunk_size(allocated_size);
Byte *user_ptr = base_ptr + shift;
- size_t shift_exponent = static_cast<size_t>(cpp::countr_zero(shift));
-
size_t *header = reinterpret_cast<size_t *>(user_ptr - HEADER_SIZE);
- *header = actual_chunk_size | (shift_exponent & 31);
+ *header = actual_chunk_size | shift_exponent;
return user_ptr;
}
@@ -444,13 +451,13 @@ class Heap {
size_t *header = reinterpret_cast<size_t *>(user_ptr - HEADER_SIZE);
size_t header_val = *header;
- size_t shift_exponent = header_val & 31;
- LIBC_ASSERT(shift_exponent >= 3 && "Invalid or double freed pointer");
- if (shift_exponent < 3)
+ size_t shift_exponent = header_val & SHIFT_MASK;
+ LIBC_ASSERT(shift_exponent >= MIN_SHIFT_EXPONENT && "Invalid or double freed pointer");
+ if (shift_exponent < MIN_SHIFT_EXPONENT)
return;
size_t shift = size_t{1} << shift_exponent;
- size_t actual_chunk_size = header_val & ~31;
+ size_t actual_chunk_size = header_val & ~SHIFT_MASK;
Byte *base_ptr = user_ptr - shift;
@@ -470,12 +477,12 @@ class Heap {
size_t *header = reinterpret_cast<size_t *>(user_ptr - HEADER_SIZE);
size_t header_val = *header;
- size_t shift_exponent = header_val & 31;
- if (shift_exponent < 3)
+ size_t shift_exponent = header_val & SHIFT_MASK;
+ if (shift_exponent < MIN_SHIFT_EXPONENT)
return nullptr;
size_t shift = size_t{1} << shift_exponent;
- size_t old_chunk_size = header_val & ~31;
+ size_t old_chunk_size = header_val & ~SHIFT_MASK;
Byte *base_ptr = user_ptr - shift;
diff --git a/libc/src/__support/flat_tlsf/tag.h b/libc/src/__support/flat_tlsf/tag.h
index 7feb540e22932..68944722d6f7f 100644
--- a/libc/src/__support/flat_tlsf/tag.h
+++ b/libc/src/__support/flat_tlsf/tag.h
@@ -37,11 +37,11 @@ LIBC_INLINE bool is_heap_end(Byte tag) { return tag & HEAP_END_FLAG; }
LIBC_INLINE void set_above_free(Byte *ptr) { *ptr |= ABOVE_FREE_FLAG; }
-LIBC_INLINE void clear_above_free(Byte *ptr) { *ptr ^= ABOVE_FREE_FLAG; }
+LIBC_INLINE void clear_above_free(Byte *ptr) { *ptr &= ~ABOVE_FREE_FLAG; }
-LIBC_INLINE void set_end_flag(Byte *ptr) { *ptr ^= HEAP_END_FLAG; }
+LIBC_INLINE void set_end_flag(Byte *ptr) { *ptr |= HEAP_END_FLAG; }
-LIBC_INLINE void clear_end_flag(Byte *ptr) { *ptr ^= HEAP_END_FLAG; }
+LIBC_INLINE void clear_end_flag(Byte *ptr) { *ptr &= ~HEAP_END_FLAG; }
} // namespace tag
} // namespace flat_tlsf
diff --git a/libc/test/src/__support/flat_tlsf/binning_test.cpp b/libc/test/src/__support/flat_tlsf/binning_test.cpp
index afae9b6d5df55..9ec3d388cc6fb 100644
--- a/libc/test/src/__support/flat_tlsf/binning_test.cpp
+++ b/libc/test/src/__support/flat_tlsf/binning_test.cpp
@@ -141,7 +141,7 @@ TEST_F(LlvmLibcFlatTlsfBinningTest,
TestLinearExtentThenLinearlyDividedExponentialBinning) {
auto size_to_bin_fn = [](size_t size) {
return static_cast<uint32_t>(
- Binning::linear_extend_then_linearly_divided_expotential_binning<8, 4>(
+ Binning::linear_extend_then_linearly_divided_exponential_binning<8, 4>(
size));
};
check_binning_properties(nullopt, size_to_bin_fn);
diff --git a/libc/test/src/__support/flat_tlsf/tag_test.cpp b/libc/test/src/__support/flat_tlsf/tag_test.cpp
index 88eb07f97db76..234aa93488f93 100644
--- a/libc/test/src/__support/flat_tlsf/tag_test.cpp
+++ b/libc/test/src/__support/flat_tlsf/tag_test.cpp
@@ -44,6 +44,16 @@ TEST(LlvmLibcFlatTlsfTagTest, MutatesFlags) {
EXPECT_TRUE(tag::is_above_free(byte));
EXPECT_TRUE(tag::is_allocated(byte));
+ // Test idempotency: setting already set flag should keep it set.
+ tag::set_above_free(&byte);
+ EXPECT_TRUE(tag::is_above_free(byte));
+ EXPECT_TRUE(tag::is_allocated(byte));
+
+ tag::clear_above_free(&byte);
+ EXPECT_FALSE(tag::is_above_free(byte));
+ EXPECT_TRUE(tag::is_allocated(byte));
+
+ // Test idempotency: clearing already cleared flag should keep it clear.
tag::clear_above_free(&byte);
EXPECT_FALSE(tag::is_above_free(byte));
EXPECT_TRUE(tag::is_allocated(byte));
@@ -52,6 +62,16 @@ TEST(LlvmLibcFlatTlsfTagTest, MutatesFlags) {
EXPECT_TRUE(tag::is_heap_end(byte));
EXPECT_TRUE(tag::is_allocated(byte));
+ // Test idempotency: setting already set flag should keep it set.
+ tag::set_end_flag(&byte);
+ EXPECT_TRUE(tag::is_heap_end(byte));
+ EXPECT_TRUE(tag::is_allocated(byte));
+
+ tag::clear_end_flag(&byte);
+ EXPECT_FALSE(tag::is_heap_end(byte));
+ EXPECT_TRUE(tag::is_allocated(byte));
+
+ // Test idempotency: clearing already cleared flag should keep it clear.
tag::clear_end_flag(&byte);
EXPECT_FALSE(tag::is_heap_end(byte));
EXPECT_TRUE(tag::is_allocated(byte));
>From 18918995cb44ce8d1f748ff0052e26e47fa6eae0 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Thu, 28 May 2026 10:02:19 -0700
Subject: [PATCH 7/8] fix
---
libc/src/__support/flat_tlsf/CMakeLists.txt | 1 +
libc/src/__support/flat_tlsf/binning.h | 4 +--
libc/src/__support/flat_tlsf/bit_utils.h | 4 +--
libc/src/__support/flat_tlsf/chunk.h | 16 ++++++++++++
libc/src/__support/flat_tlsf/heap.h | 26 ++++++++++++-------
libc/src/__support/freelist_heap.cpp | 2 +-
.../src/__support/flat_tlsf/heap_test.cpp | 23 +++++++++++++---
7 files changed, 58 insertions(+), 18 deletions(-)
diff --git a/libc/src/__support/flat_tlsf/CMakeLists.txt b/libc/src/__support/flat_tlsf/CMakeLists.txt
index b57e6ce8aec3b..f5741ce1edb24 100644
--- a/libc/src/__support/flat_tlsf/CMakeLists.txt
+++ b/libc/src/__support/flat_tlsf/CMakeLists.txt
@@ -75,6 +75,7 @@ add_header_library(
libc.src.__support.macros.attributes
libc.src.__support.macros.config
libc.src.string.memory_utils.inline_memcpy
+ libc.src.__support.common
)
add_header_library(
diff --git a/libc/src/__support/flat_tlsf/binning.h b/libc/src/__support/flat_tlsf/binning.h
index 17e390085d836..6958f53f56fc1 100644
--- a/libc/src/__support/flat_tlsf/binning.h
+++ b/libc/src/__support/flat_tlsf/binning.h
@@ -40,7 +40,7 @@ struct Binning {
/// # Behavior by size
/// - `0..=(CHUNK_UNIT*LIN_DIVS*LIN_EXT_MULTI)` : Bins sizes into
/// one-bin-per-chunk-size
- /// - `(CHUNK_UNIT*LIN_DIVS*LIN_EXT_MULTI)..` : Binds sizes by
+ /// - `(CHUNK_UNIT*LIN_DIVS*LIN_EXT_MULTI)..` : Bins sizes by
/// linearly-subdivided exponential levels.
///
/// # Parameters
@@ -97,7 +97,7 @@ struct Binning {
// Subdiv 0: 256 ; bin 0 + LIN_DIVS * LIN_EXT_MULTI
// Subdiv 1: 256 + 64 ; bin 1 + LIN_DIVS * LIN_EXT_MULTI
// Subdiv 2: 256 + 128 ; bin 2 + LIN_DIVS * LIN_EXT_MULTI
- // Subdiv 3: 256 + 196 ; bin 3 + LIN_DIVS * LIN_EXT_MULTI
+ // Subdiv 3: 256 + 192 ; bin 3 + LIN_DIVS * LIN_EXT_MULTI
// Exponential level 1: 512 ; 512/LIN_DIVS = 128
// Subdiv 0: 512 ; bin 4 + LIN_DIVS * LIN_EXT_MULTI
// Subdiv 1: 512 + 128 ; bin 5 + LIN_DIVS * LIN_EXT_MULTI
diff --git a/libc/src/__support/flat_tlsf/bit_utils.h b/libc/src/__support/flat_tlsf/bit_utils.h
index cf52c8b34b2e5..1c3734b781afe 100644
--- a/libc/src/__support/flat_tlsf/bit_utils.h
+++ b/libc/src/__support/flat_tlsf/bit_utils.h
@@ -25,8 +25,8 @@
namespace LIBC_NAMESPACE_DECL {
namespace flat_tlsf {
-// We could use cpp::byte, but that definition currently have strict aliasing
-// violation, hence we fallback to unsigned char directly.
+// We could use cpp::byte, but that definition currently has a strict aliasing
+// violation, hence we fall back to unsigned char directly.
namespace bit_utils {
LIBC_INLINE constexpr size_t ilog2(size_t n) {
diff --git a/libc/src/__support/flat_tlsf/chunk.h b/libc/src/__support/flat_tlsf/chunk.h
index 62cfa56f62799..7da87e83a7de5 100644
--- a/libc/src/__support/flat_tlsf/chunk.h
+++ b/libc/src/__support/flat_tlsf/chunk.h
@@ -20,6 +20,7 @@
#include "src/__support/macros/attributes.h"
#include "src/__support/macros/config.h"
#include "src/string/memory_utils/inline_memcpy.h"
+#include "src/__support/endian_internal.h"
namespace LIBC_NAMESPACE_DECL {
namespace flat_tlsf {
@@ -51,6 +52,13 @@ LIBC_INLINE size_t *gap_base_to_size(Byte *base) {
return reinterpret_cast<size_t *>(base + GAP_LOW_SIZE_OFFSET);
}
+// Note: The highest-addressed byte of this size word (at `end - 1`) overlaps
+// with the tag byte of the next chunk (retrieved via `end_to_tag(end)`).
+// To prevent any tag/flag corruption or endianness issues, the size word is
+// always stored and read in Big-Endian format. This guarantees that the LSB
+// of the size word is always at the highest address (`end - 1`). Since all
+// gap sizes are CHUNK_UNIT aligned, the lower bits of the LSB at `end - 1` are
+// naturally 0, keeping the tag/flag status bits safely at 0 (free gap).
LIBC_INLINE size_t *gap_end_to_size_and_flag(Byte *end) {
return reinterpret_cast<size_t *>(end - GAP_HIGH_SIZE_OFFSET);
}
@@ -84,6 +92,14 @@ template <class T> LIBC_INLINE void write_word(void *ptr, T value) {
inline_memcpy(ptr, &value, sizeof(T));
}
+template <class T> LIBC_INLINE T read_big_endian(const void *ptr) {
+ return Endian::to_big_endian(read_word<T>(ptr));
+}
+
+template <class T> LIBC_INLINE void write_big_endian(void *ptr, T value) {
+ write_word<T>(ptr, Endian::to_big_endian(value));
+}
+
} // namespace chunk
} // namespace flat_tlsf
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/flat_tlsf/heap.h b/libc/src/__support/flat_tlsf/heap.h
index 6479767c1cf90..ae7f6d4e9809e 100644
--- a/libc/src/__support/flat_tlsf/heap.h
+++ b/libc/src/__support/flat_tlsf/heap.h
@@ -46,7 +46,7 @@ class Heap {
LIBC_INLINE cpp::optional<SearchResult>
full_search_bin(uint32_t bin, size_t required_size, size_t align_mask) {
for (Node *node = gap_list[bin]; node != nullptr; node = node->next) {
- size_t size = chunk::read_word<size_t>(chunk::gap_node_to_size(node));
+ size_t size = chunk::read_big_endian<size_t>(chunk::gap_node_to_size(node));
Byte *base = chunk::gap_node_to_base(node);
Byte *end = base + size;
@@ -79,8 +79,8 @@ class Heap {
chunk::gap_base_to_node(base)->link_at(Node{*bin_ptr, bin_ptr});
chunk::write_word(chunk::gap_base_to_bin(base), bin);
- chunk::write_word(chunk::gap_base_to_size(base), size);
- chunk::write_word(chunk::gap_end_to_size_and_flag(end), size);
+ chunk::write_big_endian(chunk::gap_base_to_size(base), size);
+ chunk::write_big_endian(chunk::gap_end_to_size_and_flag(end), size);
LIBC_ASSERT(*bin_ptr != nullptr);
}
@@ -166,6 +166,8 @@ class Heap {
}
LIBC_INLINE Byte *allocate(size_t required_size, size_t required_align) {
+ if (required_size > cpp::numeric_limits<size_t>::max() - CHUNK_UNIT)
+ return nullptr;
size_t required_chunk_size = chunk::required_chunk_size(required_size);
Byte *base = nullptr;
Byte *chunk_end = nullptr;
@@ -203,7 +205,7 @@ class Heap {
if (required_align <= CHUNK_UNIT) {
Node *node_ptr = gap_list[bit];
size_t size =
- chunk::read_word<size_t>(chunk::gap_node_to_size(node_ptr));
+ chunk::read_big_endian<size_t>(chunk::gap_node_to_size(node_ptr));
LIBC_ASSERT(size >= required_chunk_size);
base = chunk::gap_node_to_base(node_ptr);
@@ -217,7 +219,7 @@ class Heap {
while (true) {
for (Node *node = gap_list[bit]; node != nullptr; node = node->next) {
size_t size =
- chunk::read_word<size_t>(chunk::gap_node_to_size(node));
+ chunk::read_big_endian<size_t>(chunk::gap_node_to_size(node));
Byte *b = chunk::gap_node_to_base(node);
Byte *end = b + size;
Byte *aligned_base = bit_utils::align_up_by_mask(b, align_mask);
@@ -248,7 +250,7 @@ class Heap {
for (Node *node = gap_list[bin - 1]; node != nullptr;
node = node->next) {
size_t size =
- chunk::read_word<size_t>(chunk::gap_node_to_size(node));
+ chunk::read_big_endian<size_t>(chunk::gap_node_to_size(node));
Byte *b = chunk::gap_node_to_base(node);
Byte *end = b + size;
Byte *aligned_base = bit_utils::align_up_by_mask(b, align_mask);
@@ -297,7 +299,7 @@ class Heap {
Byte *below_tag_ptr = chunk::end_to_tag(chunk_base);
if (!tag::is_allocated(chunk::read_word<Byte>(below_tag_ptr))) {
size_t below_size =
- chunk::read_word<size_t>(chunk::gap_end_to_size_and_flag(chunk_base));
+ chunk::read_big_endian<size_t>(chunk::gap_end_to_size_and_flag(chunk_base));
Byte *below_base = chunk_base - below_size;
deregister_gap(below_base, below_size);
@@ -312,7 +314,7 @@ class Heap {
if (tag::is_above_free(tag)) {
LIBC_ASSERT(!tag::is_heap_end(tag));
size_t above_size =
- chunk::read_word<size_t>(chunk::gap_base_to_size(chunk_end));
+ chunk::read_big_endian<size_t>(chunk::gap_base_to_size(chunk_end));
deregister_gap(chunk_end, above_size);
chunk_end += above_size;
}
@@ -335,7 +337,7 @@ class Heap {
if (tag::is_above_free(old_tag)) {
size_t above_size =
- chunk::read_word<size_t>(chunk::gap_base_to_size(old_end));
+ chunk::read_big_endian<size_t>(chunk::gap_base_to_size(old_end));
Byte *above_end = old_end + above_size;
if (new_end <= above_end) {
@@ -371,7 +373,7 @@ class Heap {
if (tag::is_above_free(old_tag)) {
size_t above_size =
- chunk::read_word<size_t>(chunk::gap_base_to_size(chunk_end));
+ chunk::read_big_endian<size_t>(chunk::gap_base_to_size(chunk_end));
deregister_gap(chunk_end, above_size);
chunk_end += above_size;
}
@@ -428,6 +430,8 @@ class Heap {
if (shift_exponent > SHIFT_MASK)
return nullptr;
+ if (size > cpp::numeric_limits<size_t>::max() - shift - CHUNK_UNIT)
+ return nullptr;
size_t allocated_size = size + shift;
Byte *base_ptr = allocate(allocated_size, allocated_align);
@@ -486,6 +490,8 @@ class Heap {
Byte *base_ptr = user_ptr - shift;
+ if (new_size > cpp::numeric_limits<size_t>::max() - shift - CHUNK_UNIT)
+ return nullptr;
size_t new_allocated_size = new_size + shift;
size_t new_chunk_size = chunk::required_chunk_size(new_allocated_size);
diff --git a/libc/src/__support/freelist_heap.cpp b/libc/src/__support/freelist_heap.cpp
index 4deb0e0f09e22..e3a6f4f3d3efe 100644
--- a/libc/src/__support/freelist_heap.cpp
+++ b/libc/src/__support/freelist_heap.cpp
@@ -13,7 +13,7 @@
namespace LIBC_NAMESPACE_DECL {
-static LIBC_CONSTINIT FreeListHeap freelist_heap_symbols;
+LIBC_CONSTINIT FreeListHeap freelist_heap_symbols;
FreeListHeap *freelist_heap = &freelist_heap_symbols;
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/flat_tlsf/heap_test.cpp b/libc/test/src/__support/flat_tlsf/heap_test.cpp
index fc789281e5476..226cbbf1b800a 100644
--- a/libc/test/src/__support/flat_tlsf/heap_test.cpp
+++ b/libc/test/src/__support/flat_tlsf/heap_test.cpp
@@ -92,13 +92,13 @@ TEST(LlvmLibcFlatTlsfHeapTest, VerifyGapProperties) {
ASSERT_EQ(gap_bin,
chunk::read_word<uint32_t>(chunk::gap_base_to_bin(gap_base)));
ASSERT_EQ(gap_size,
- chunk::read_word<size_t>(chunk::gap_base_to_size(gap_base)));
+ chunk::read_big_endian<size_t>(chunk::gap_base_to_size(gap_base)));
- ASSERT_EQ(chunk::read_word<size_t>(chunk::gap_base_to_size(gap_base)),
+ ASSERT_EQ(chunk::read_big_endian<size_t>(chunk::gap_base_to_size(gap_base)),
gap_size);
ASSERT_EQ(chunk::gap_end_to_size_and_flag(gap_end),
reinterpret_cast<size_t *>(gap_end - sizeof(size_t)));
- ASSERT_EQ(chunk::read_word<size_t>(chunk::gap_end_to_size_and_flag(gap_end)),
+ ASSERT_EQ(chunk::read_big_endian<size_t>(chunk::gap_end_to_size_and_flag(gap_end)),
gap_size);
heap.test_deregister_gap(gap_base, gap_size);
@@ -135,6 +135,23 @@ TEST(LlvmLibcFlatTlsfHeapTest, AllocFailTest) {
ASSERT_EQ(a2, static_cast<Byte *>(nullptr));
}
+TEST(LlvmLibcFlatTlsfHeapTest, AllocOverflowTest) {
+ alignas(64) Byte arena[5000] = {};
+ Heap heap;
+ ASSERT_NE(heap.claim(arena, 5000), static_cast<Byte *>(nullptr));
+
+ // Test malloc overflow
+ ASSERT_EQ(heap.malloc(numeric_limits<size_t>::max()), static_cast<void *>(nullptr));
+
+ // Test aligned_alloc overflow
+ ASSERT_EQ(heap.aligned_alloc(8, numeric_limits<size_t>::max() - 100), static_cast<void *>(nullptr));
+
+ // Test realloc overflow
+ void *ptr = heap.malloc(10);
+ ASSERT_NE(ptr, static_cast<void *>(nullptr));
+ ASSERT_EQ(heap.realloc(ptr, numeric_limits<size_t>::max()), static_cast<void *>(nullptr));
+}
+
TEST(LlvmLibcFlatTlsfHeapTest, ClaimHeapThatsTooSmall) {
alignas(8) Byte tiny_heap[200] = {};
Heap heap;
>From 24fdbe036c647c339398ccfdfc34b6c6912a5c23 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yfzhu at google.com>
Date: Thu, 28 May 2026 10:10:21 -0700
Subject: [PATCH 8/8] fix
---
libc/src/__support/flat_tlsf/chunk.h | 2 +-
libc/src/__support/flat_tlsf/heap.h | 13 ++++++++-----
libc/test/src/__support/flat_tlsf/heap_test.cpp | 14 +++++++++-----
3 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/libc/src/__support/flat_tlsf/chunk.h b/libc/src/__support/flat_tlsf/chunk.h
index 7da87e83a7de5..2fa92e3a92084 100644
--- a/libc/src/__support/flat_tlsf/chunk.h
+++ b/libc/src/__support/flat_tlsf/chunk.h
@@ -14,13 +14,13 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_CHUNK_H
#define LLVM_LIBC_SRC___SUPPORT_FLAT_TLSF_CHUNK_H
+#include "src/__support/endian_internal.h"
#include "src/__support/flat_tlsf/bit_utils.h"
#include "src/__support/flat_tlsf/common.h"
#include "src/__support/flat_tlsf/node.h"
#include "src/__support/macros/attributes.h"
#include "src/__support/macros/config.h"
#include "src/string/memory_utils/inline_memcpy.h"
-#include "src/__support/endian_internal.h"
namespace LIBC_NAMESPACE_DECL {
namespace flat_tlsf {
diff --git a/libc/src/__support/flat_tlsf/heap.h b/libc/src/__support/flat_tlsf/heap.h
index ae7f6d4e9809e..8219c0a571ced 100644
--- a/libc/src/__support/flat_tlsf/heap.h
+++ b/libc/src/__support/flat_tlsf/heap.h
@@ -46,7 +46,8 @@ class Heap {
LIBC_INLINE cpp::optional<SearchResult>
full_search_bin(uint32_t bin, size_t required_size, size_t align_mask) {
for (Node *node = gap_list[bin]; node != nullptr; node = node->next) {
- size_t size = chunk::read_big_endian<size_t>(chunk::gap_node_to_size(node));
+ size_t size =
+ chunk::read_big_endian<size_t>(chunk::gap_node_to_size(node));
Byte *base = chunk::gap_node_to_base(node);
Byte *end = base + size;
@@ -298,8 +299,8 @@ class Heap {
// the presence of an end flag.
Byte *below_tag_ptr = chunk::end_to_tag(chunk_base);
if (!tag::is_allocated(chunk::read_word<Byte>(below_tag_ptr))) {
- size_t below_size =
- chunk::read_big_endian<size_t>(chunk::gap_end_to_size_and_flag(chunk_base));
+ size_t below_size = chunk::read_big_endian<size_t>(
+ chunk::gap_end_to_size_and_flag(chunk_base));
Byte *below_base = chunk_base - below_size;
deregister_gap(below_base, below_size);
@@ -426,7 +427,8 @@ class Heap {
size_t shift = (HEADER_SIZE + align - 1) & ~(align - 1);
size_t shift_exponent = static_cast<size_t>(cpp::countr_zero(shift));
- LIBC_ASSERT(shift_exponent <= SHIFT_MASK && "Alignment too large for allocation header");
+ LIBC_ASSERT(shift_exponent <= SHIFT_MASK &&
+ "Alignment too large for allocation header");
if (shift_exponent > SHIFT_MASK)
return nullptr;
@@ -456,7 +458,8 @@ class Heap {
size_t header_val = *header;
size_t shift_exponent = header_val & SHIFT_MASK;
- LIBC_ASSERT(shift_exponent >= MIN_SHIFT_EXPONENT && "Invalid or double freed pointer");
+ LIBC_ASSERT(shift_exponent >= MIN_SHIFT_EXPONENT &&
+ "Invalid or double freed pointer");
if (shift_exponent < MIN_SHIFT_EXPONENT)
return;
diff --git a/libc/test/src/__support/flat_tlsf/heap_test.cpp b/libc/test/src/__support/flat_tlsf/heap_test.cpp
index 226cbbf1b800a..a0da2f655164d 100644
--- a/libc/test/src/__support/flat_tlsf/heap_test.cpp
+++ b/libc/test/src/__support/flat_tlsf/heap_test.cpp
@@ -98,8 +98,9 @@ TEST(LlvmLibcFlatTlsfHeapTest, VerifyGapProperties) {
gap_size);
ASSERT_EQ(chunk::gap_end_to_size_and_flag(gap_end),
reinterpret_cast<size_t *>(gap_end - sizeof(size_t)));
- ASSERT_EQ(chunk::read_big_endian<size_t>(chunk::gap_end_to_size_and_flag(gap_end)),
- gap_size);
+ ASSERT_EQ(
+ chunk::read_big_endian<size_t>(chunk::gap_end_to_size_and_flag(gap_end)),
+ gap_size);
heap.test_deregister_gap(gap_base, gap_size);
}
@@ -141,15 +142,18 @@ TEST(LlvmLibcFlatTlsfHeapTest, AllocOverflowTest) {
ASSERT_NE(heap.claim(arena, 5000), static_cast<Byte *>(nullptr));
// Test malloc overflow
- ASSERT_EQ(heap.malloc(numeric_limits<size_t>::max()), static_cast<void *>(nullptr));
+ ASSERT_EQ(heap.malloc(numeric_limits<size_t>::max()),
+ static_cast<void *>(nullptr));
// Test aligned_alloc overflow
- ASSERT_EQ(heap.aligned_alloc(8, numeric_limits<size_t>::max() - 100), static_cast<void *>(nullptr));
+ ASSERT_EQ(heap.aligned_alloc(8, numeric_limits<size_t>::max() - 100),
+ static_cast<void *>(nullptr));
// Test realloc overflow
void *ptr = heap.malloc(10);
ASSERT_NE(ptr, static_cast<void *>(nullptr));
- ASSERT_EQ(heap.realloc(ptr, numeric_limits<size_t>::max()), static_cast<void *>(nullptr));
+ ASSERT_EQ(heap.realloc(ptr, numeric_limits<size_t>::max()),
+ static_cast<void *>(nullptr));
}
TEST(LlvmLibcFlatTlsfHeapTest, ClaimHeapThatsTooSmall) {
More information about the libc-commits
mailing list