[libc-commits] [libc] [WIP][libc] Add freelist malloc (PR #94270)
via libc-commits
libc-commits at lists.llvm.org
Mon Jun 10 13:40:41 PDT 2024
https://github.com/PiJoules updated https://github.com/llvm/llvm-project/pull/94270
>From 2779336c1415d6dfcbaadccb7d6d02825f95df79 Mon Sep 17 00:00:00 2001
From: Leonard Chan <leonardchan at google.com>
Date: Fri, 31 May 2024 14:27:16 -0700
Subject: [PATCH] [WIP][libc] Add freelist malloc
---
libc/config/baremetal/riscv/entrypoints.txt | 1 +
libc/src/stdlib/CMakeLists.txt | 15 +-
libc/src/stdlib/block.h | 4 +-
libc/src/stdlib/free.h | 2 +-
libc/src/stdlib/freelist.h | 203 +++++++++++++++
libc/src/stdlib/freelist_heap.h | 208 +++++++++++++++
libc/src/stdlib/freelist_malloc.cpp | 42 +++
libc/test/src/CMakeLists.txt | 2 +-
libc/test/src/stdlib/CMakeLists.txt | 31 +--
libc/test/src/stdlib/freelist_heap_test.cpp | 241 ++++++++++++++++++
libc/test/src/stdlib/freelist_malloc_test.cpp | 56 ++++
libc/test/src/stdlib/freelist_test.cpp | 166 ++++++++++++
libc/test/src/stdlib/malloc_test.cpp | 4 +
13 files changed, 955 insertions(+), 20 deletions(-)
create mode 100644 libc/src/stdlib/freelist.h
create mode 100644 libc/src/stdlib/freelist_heap.h
create mode 100644 libc/src/stdlib/freelist_malloc.cpp
create mode 100644 libc/test/src/stdlib/freelist_heap_test.cpp
create mode 100644 libc/test/src/stdlib/freelist_malloc_test.cpp
create mode 100644 libc/test/src/stdlib/freelist_test.cpp
diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt
index b769b43f03a2c..363a762909c3a 100644
--- a/libc/config/baremetal/riscv/entrypoints.txt
+++ b/libc/config/baremetal/riscv/entrypoints.txt
@@ -170,6 +170,7 @@ set(TARGET_LIBC_ENTRYPOINTS
libc.src.stdlib.ldiv
libc.src.stdlib.llabs
libc.src.stdlib.lldiv
+ libc.src.stdlib.malloc
libc.src.stdlib.qsort
libc.src.stdlib.rand
libc.src.stdlib.srand
diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt
index afb2d6d91cba4..89b24276539f2 100644
--- a/libc/src/stdlib/CMakeLists.txt
+++ b/libc/src/stdlib/CMakeLists.txt
@@ -392,8 +392,21 @@ else()
libc.src.__support.CPP.span
libc.src.__support.CPP.type_traits
)
- add_entrypoint_external(
+ add_entrypoint_object(
malloc
+ SRCS
+ freelist_malloc.cpp
+ HDRS
+ malloc.h
+ DEPENDS
+ .block
+ libc.src.__support.CPP.new
+ libc.src.__support.CPP.optional
+ libc.src.__support.CPP.span
+ libc.src.__support.CPP.type_traits
+ libc.src.__support.fixedvector
+ libc.src.string.memcpy
+ libc.src.string.memset
)
add_entrypoint_external(
free
diff --git a/libc/src/stdlib/block.h b/libc/src/stdlib/block.h
index ce26add082d0a..afb18c1ef738f 100644
--- a/libc/src/stdlib/block.h
+++ b/libc/src/stdlib/block.h
@@ -39,8 +39,8 @@ LIBC_INLINE constexpr size_t align_down(size_t value, size_t alignment) {
}
/// Returns the value rounded down to the nearest multiple of alignment.
-LIBC_INLINE template <typename T>
-constexpr T *align_down(T *value, size_t alignment) {
+template <typename T>
+LIBC_INLINE constexpr T *align_down(T *value, size_t alignment) {
return reinterpret_cast<T *>(
align_down(reinterpret_cast<size_t>(value), alignment));
}
diff --git a/libc/src/stdlib/free.h b/libc/src/stdlib/free.h
index f802f1d192d81..b3970fd967740 100644
--- a/libc/src/stdlib/free.h
+++ b/libc/src/stdlib/free.h
@@ -17,4 +17,4 @@ void free(void *ptr);
} // namespace LIBC_NAMESPACE
-#endif // LLVM_LIBC_SRC_STDLIB_LDIV_H
+#endif // LLVM_LIBC_SRC_STDLIB_FREE_H
diff --git a/libc/src/stdlib/freelist.h b/libc/src/stdlib/freelist.h
new file mode 100644
index 0000000000000..ee53b6eb8a4e1
--- /dev/null
+++ b/libc/src/stdlib/freelist.h
@@ -0,0 +1,203 @@
+//===-- Interface for freelist_malloc -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDLIB_FREELIST_H
+#define LLVM_LIBC_SRC_STDLIB_FREELIST_H
+
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/cstddef.h"
+#include "src/__support/CPP/span.h"
+#include "src/__support/fixedvector.h"
+
+namespace LIBC_NAMESPACE {
+
+using cpp::span;
+
+/// Basic [freelist](https://en.wikipedia.org/wiki/Free_list) implementation
+/// for an allocator. This implementation buckets by chunk size, with a list
+/// of user-provided buckets. Each bucket is a linked list of storage chunks.
+/// Because this freelist uses the added chunks themselves as list nodes, there
+/// is a lower bound of `sizeof(FreeList.FreeListNode)` bytes for chunks which
+/// can be added to this freelist. There is also an implicit bucket for
+/// "everything else", for chunks which do not fit into a bucket.
+///
+/// Each added chunk will be added to the smallest bucket under which it fits.
+/// If it does not fit into any user-provided bucket, it will be added to the
+/// default bucket.
+///
+/// As an example, assume that the `FreeList` is configured with buckets of
+/// sizes {64, 128, 256, and 512} bytes. The internal state may look like the
+/// following:
+///
+/// @code{.unparsed}
+/// bucket[0] (64B) --> chunk[12B] --> chunk[42B] --> chunk[64B] --> NULL
+/// bucket[1] (128B) --> chunk[65B] --> chunk[72B] --> NULL
+/// bucket[2] (256B) --> NULL
+/// bucket[3] (512B) --> chunk[312B] --> chunk[512B] --> chunk[416B] --> NULL
+/// bucket[4] (implicit) --> chunk[1024B] --> chunk[513B] --> NULL
+/// @endcode
+///
+/// Note that added chunks should be aligned to a 4-byte boundary.
+template <size_t kNumBuckets = 6> class FreeList {
+public:
+ // Remove copy/move ctors
+ FreeList(const FreeList &other) = delete;
+ FreeList(FreeList &&other) = delete;
+ FreeList &operator=(const FreeList &other) = delete;
+ FreeList &operator=(FreeList &&other) = delete;
+
+ /// Adds a chunk to this freelist.
+ bool AddChunk(cpp::span<cpp::byte> chunk);
+
+ /// Finds an eligible chunk for an allocation of size `size`.
+ ///
+ /// @note This returns the first allocation possible within a given bucket;
+ /// It does not currently optimize for finding the smallest chunk.
+ ///
+ /// @returns
+ /// * On success - A span representing the chunk.
+ /// * On failure (e.g. there were no chunks available for that allocation) -
+ /// A span with a size of 0.
+ cpp::span<cpp::byte> FindChunk(size_t size) const;
+
+ /// Removes a chunk from this freelist.
+ bool RemoveChunk(cpp::span<cpp::byte> chunk);
+
+private:
+ // For a given size, find which index into chunks_ the node should be written
+ // to.
+ unsigned short FindChunkPtrForSize(size_t size, bool non_null) const;
+
+ struct FreeListNode {
+ FreeListNode *next;
+ size_t size;
+ };
+
+public:
+ explicit FreeList(cpp::array<size_t, kNumBuckets> sizes)
+ : chunks_(kNumBuckets + 1, 0), sizes_(sizes.begin(), sizes.end()) {}
+
+ FixedVector<FreeList::FreeListNode *, kNumBuckets + 1> chunks_;
+ FixedVector<size_t, kNumBuckets> sizes_;
+};
+
+template <size_t kNumBuckets>
+bool FreeList<kNumBuckets>::AddChunk(span<cpp::byte> chunk) {
+ // Check that the size is enough to actually store what we need
+ if (chunk.size() < sizeof(FreeListNode)) {
+ return false;
+ }
+
+ union {
+ FreeListNode *node;
+ cpp::byte *bytes;
+ } aliased;
+
+ aliased.bytes = chunk.data();
+
+ unsigned short chunk_ptr = FindChunkPtrForSize(chunk.size(), false);
+
+ // Add it to the correct list.
+ aliased.node->size = chunk.size();
+ aliased.node->next = chunks_[chunk_ptr];
+ chunks_[chunk_ptr] = aliased.node;
+
+ return true;
+}
+
+template <size_t kNumBuckets>
+span<cpp::byte> FreeList<kNumBuckets>::FindChunk(size_t size) const {
+ if (size == 0) {
+ return span<cpp::byte>();
+ }
+
+ unsigned short chunk_ptr = FindChunkPtrForSize(size, true);
+
+ // Check that there's data. This catches the case where we run off the
+ // end of the array
+ if (chunks_[chunk_ptr] == nullptr) {
+ return span<cpp::byte>();
+ }
+
+ // Now iterate up the buckets, walking each list to find a good candidate
+ for (size_t i = chunk_ptr; i < chunks_.size(); i++) {
+ union {
+ FreeListNode *node;
+ cpp::byte *data;
+ } aliased;
+ aliased.node = chunks_[static_cast<unsigned short>(i)];
+
+ while (aliased.node != nullptr) {
+ if (aliased.node->size >= size) {
+ return span<cpp::byte>(aliased.data, aliased.node->size);
+ }
+
+ aliased.node = aliased.node->next;
+ }
+ }
+
+ // If we get here, we've checked every block in every bucket. There's
+ // nothing that can support this allocation.
+ return span<cpp::byte>();
+}
+
+template <size_t kNumBuckets>
+bool FreeList<kNumBuckets>::RemoveChunk(span<cpp::byte> chunk) {
+ unsigned short chunk_ptr = FindChunkPtrForSize(chunk.size(), true);
+
+ // Walk that list, finding the chunk.
+ union {
+ FreeListNode *node;
+ cpp::byte *data;
+ } aliased, aliased_next;
+
+ // Check head first.
+ if (chunks_[chunk_ptr] == nullptr) {
+ return false;
+ }
+
+ aliased.node = chunks_[chunk_ptr];
+ if (aliased.data == chunk.data()) {
+ chunks_[chunk_ptr] = aliased.node->next;
+ return true;
+ }
+
+ // No? Walk the nodes.
+ aliased.node = chunks_[chunk_ptr];
+
+ while (aliased.node->next != nullptr) {
+ aliased_next.node = aliased.node->next;
+ if (aliased_next.data == chunk.data()) {
+ // Found it, remove this node out of the chain
+ aliased.node->next = aliased_next.node->next;
+ return true;
+ }
+
+ aliased.node = aliased.node->next;
+ }
+
+ return false;
+}
+
+template <size_t kNumBuckets>
+unsigned short FreeList<kNumBuckets>::FindChunkPtrForSize(size_t size,
+ bool non_null) const {
+ unsigned short chunk_ptr = 0;
+ for (chunk_ptr = 0u; chunk_ptr < sizes_.size(); chunk_ptr++) {
+ if (sizes_[chunk_ptr] >= size &&
+ (!non_null || chunks_[chunk_ptr] != nullptr)) {
+ break;
+ }
+ }
+
+ return chunk_ptr;
+}
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_STDLIB_FREELIST_H
diff --git a/libc/src/stdlib/freelist_heap.h b/libc/src/stdlib/freelist_heap.h
new file mode 100644
index 0000000000000..f9b1830f405ed
--- /dev/null
+++ b/libc/src/stdlib/freelist_heap.h
@@ -0,0 +1,208 @@
+//===-- Interface for freelist_heap ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDLIB_FREELIST_HEAP_H
+#define LLVM_LIBC_SRC_STDLIB_FREELIST_HEAP_H
+
+#include <stddef.h>
+
+#include "block.h"
+#include "freelist.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/CPP/span.h"
+#include "src/string/memcpy.h"
+#include "src/string/memset.h"
+
+namespace LIBC_NAMESPACE {
+
+void MallocInit(uint8_t *heap_low_addr, uint8_t *heap_high_addr);
+
+using cpp::optional;
+using cpp::span;
+
+static constexpr cpp::array<size_t, 6> defaultBuckets{16, 32, 64,
+ 128, 256, 512};
+
+template <size_t kNumBuckets = defaultBuckets.size()> class FreeListHeap {
+public:
+ using BlockType = Block<>;
+
+ template <size_t> friend class FreeListHeapBuffer;
+
+ struct HeapStats {
+ size_t total_bytes;
+ size_t bytes_allocated;
+ size_t cumulative_allocated;
+ size_t cumulative_freed;
+ size_t total_allocate_calls;
+ size_t total_free_calls;
+ };
+ FreeListHeap(span<cpp::byte> region);
+
+ void *Allocate(size_t size);
+ void Free(void *ptr);
+ void *Realloc(void *ptr, size_t size);
+ void *Calloc(size_t num, size_t size);
+
+ void LogHeapStats();
+ const HeapStats &heap_stats() const { return heap_stats_; }
+
+private:
+ span<cpp::byte> BlockToSpan(BlockType *block) {
+ return span<cpp::byte>(block->usable_space(), block->inner_size());
+ }
+
+ void InvalidFreeCrash() { __builtin_trap(); }
+
+ span<cpp::byte> region_;
+ FreeList<kNumBuckets> freelist_;
+ HeapStats heap_stats_;
+};
+
+template <size_t kNumBuckets>
+FreeListHeap<kNumBuckets>::FreeListHeap(span<cpp::byte> region)
+ : freelist_(defaultBuckets), heap_stats_() {
+ auto result = BlockType::init(region);
+ BlockType *block = *result;
+
+ freelist_.AddChunk(BlockToSpan(block));
+
+ region_ = region;
+ heap_stats_.total_bytes = region.size();
+}
+
+template <size_t kNumBuckets>
+void *FreeListHeap<kNumBuckets>::Allocate(size_t size) {
+ // Find a chunk in the freelist. Split it if needed, then return
+
+ auto chunk = freelist_.FindChunk(size);
+
+ if (chunk.data() == nullptr) {
+ return nullptr;
+ }
+ freelist_.RemoveChunk(chunk);
+
+ BlockType *chunk_block = BlockType::from_usable_space(chunk.data());
+
+ // Split that chunk. If there's a leftover chunk, add it to the freelist
+ optional<BlockType *> result = BlockType::split(chunk_block, size);
+ if (result) {
+ freelist_.AddChunk(BlockToSpan(*result));
+ }
+
+ chunk_block->mark_used();
+
+ heap_stats_.bytes_allocated += size;
+ heap_stats_.cumulative_allocated += size;
+ heap_stats_.total_allocate_calls += 1;
+
+ return chunk_block->usable_space();
+}
+
+template <size_t kNumBuckets> void FreeListHeap<kNumBuckets>::Free(void *ptr) {
+ cpp::byte *bytes = static_cast<cpp::byte *>(ptr);
+
+ if (bytes < region_.data() || bytes >= region_.data() + region_.size()) {
+ InvalidFreeCrash();
+ return;
+ }
+
+ BlockType *chunk_block = BlockType::from_usable_space(bytes);
+
+ size_t size_freed = chunk_block->inner_size();
+ // Ensure that the block is in-use
+ if (!chunk_block->used()) {
+ InvalidFreeCrash();
+ return;
+ }
+ chunk_block->mark_free();
+ // Can we combine with the left or right blocks?
+ BlockType *prev = chunk_block->prev();
+ BlockType *next = nullptr;
+
+ if (!chunk_block->last()) {
+ next = chunk_block->next();
+ }
+
+ if (prev != nullptr && !prev->used()) {
+ // Remove from freelist and merge
+ freelist_.RemoveChunk(BlockToSpan(prev));
+ chunk_block = chunk_block->prev();
+ BlockType::merge_next(chunk_block);
+ }
+
+ if (next != nullptr && !next->used()) {
+ freelist_.RemoveChunk(BlockToSpan(next));
+ BlockType::merge_next(chunk_block);
+ }
+ // Add back to the freelist
+ freelist_.AddChunk(BlockToSpan(chunk_block));
+
+ heap_stats_.bytes_allocated -= size_freed;
+ heap_stats_.cumulative_freed += size_freed;
+ heap_stats_.total_free_calls += 1;
+}
+
+// Follows constract of the C standard realloc() function
+// If ptr is free'd, will return nullptr.
+template <size_t kNumBuckets>
+void *FreeListHeap<kNumBuckets>::Realloc(void *ptr, size_t size) {
+ if (size == 0) {
+ Free(ptr);
+ return nullptr;
+ }
+
+ // If the pointer is nullptr, allocate a new memory.
+ if (ptr == nullptr) {
+ return Allocate(size);
+ }
+
+ cpp::byte *bytes = static_cast<cpp::byte *>(ptr);
+
+ // TODO(chenghanzh): Enhance with debug information for out-of-range and more.
+ if (bytes < region_.data() || bytes >= region_.data() + region_.size()) {
+ return nullptr;
+ }
+
+ BlockType *chunk_block = BlockType::from_usable_space(bytes);
+ if (!chunk_block->used()) {
+ return nullptr;
+ }
+ size_t old_size = chunk_block->inner_size();
+
+ // Do nothing and return ptr if the required memory size is smaller than
+ // the current size.
+ if (old_size >= size) {
+ return ptr;
+ }
+
+ void *new_ptr = Allocate(size);
+ // Don't invalidate ptr if Allocate(size) fails to initilize the memory.
+ if (new_ptr == nullptr) {
+ return nullptr;
+ }
+ memcpy(new_ptr, ptr, old_size);
+
+ Free(ptr);
+ return new_ptr;
+}
+
+template <size_t kNumBuckets>
+void *FreeListHeap<kNumBuckets>::Calloc(size_t num, size_t size) {
+ void *ptr = Allocate(num * size);
+ if (ptr != nullptr) {
+ memset(ptr, 0, num * size);
+ }
+ return ptr;
+}
+
+extern FreeListHeap<> *freelist_heap;
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_STDLIB_FREELIST_HEAP_H
diff --git a/libc/src/stdlib/freelist_malloc.cpp b/libc/src/stdlib/freelist_malloc.cpp
new file mode 100644
index 0000000000000..ee0c9aae771eb
--- /dev/null
+++ b/libc/src/stdlib/freelist_malloc.cpp
@@ -0,0 +1,42 @@
+//===-- Implementation for freelist_malloc --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "freelist_heap.h"
+#include "src/__support/CPP/new.h"
+#include "src/__support/CPP/span.h"
+#include "src/__support/CPP/type_traits.h"
+#include "src/string/memcpy.h"
+#include "src/string/memset.h"
+
+#include <stddef.h>
+
+namespace LIBC_NAMESPACE {
+
+namespace {
+cpp::aligned_storage_t<sizeof(FreeListHeap<>), alignof(FreeListHeap<>)> buf;
+} // namespace
+
+FreeListHeap<> *freelist_heap;
+
+// Define the global heap variables.
+void MallocInit(uint8_t *heap_low_addr, uint8_t *heap_high_addr) {
+ cpp::span<LIBC_NAMESPACE::cpp::byte> allocator_freelist_raw_heap =
+ cpp::span<cpp::byte>(reinterpret_cast<cpp::byte *>(heap_low_addr),
+ heap_high_addr - heap_low_addr);
+ freelist_heap = new (&buf) FreeListHeap<>(allocator_freelist_raw_heap);
+}
+
+void *malloc(size_t size) { return freelist_heap->Allocate(size); }
+
+void free(void *ptr) { freelist_heap->Free(ptr); }
+
+void *calloc(size_t num, size_t size) {
+ return freelist_heap->Calloc(num, size);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/test/src/CMakeLists.txt b/libc/test/src/CMakeLists.txt
index a5e7a2a4dee72..935feb59ecdf6 100644
--- a/libc/test/src/CMakeLists.txt
+++ b/libc/test/src/CMakeLists.txt
@@ -61,7 +61,7 @@ add_subdirectory(inttypes)
if(${LIBC_TARGET_OS} STREQUAL "linux")
add_subdirectory(fcntl)
add_subdirectory(sched)
- add_subdirectory(sys)
+ #add_subdirectory(sys)
add_subdirectory(termios)
add_subdirectory(unistd)
endif()
diff --git a/libc/test/src/stdlib/CMakeLists.txt b/libc/test/src/stdlib/CMakeLists.txt
index f122cd56a6060..af2fa1feff8c4 100644
--- a/libc/test/src/stdlib/CMakeLists.txt
+++ b/libc/test/src/stdlib/CMakeLists.txt
@@ -412,19 +412,20 @@ if(LLVM_LIBC_FULL_BUILD)
libc.src.stdlib.quick_exit
)
- # Only the GPU has an in-tree 'malloc' implementation.
- if(LIBC_TARGET_OS_IS_GPU)
- add_libc_test(
- malloc_test
- HERMETIC_TEST_ONLY
- SUITE
- libc-stdlib-tests
- SRCS
- malloc_test.cpp
- DEPENDS
- libc.include.stdlib
- libc.src.stdlib.malloc
- libc.src.stdlib.free
- )
- endif()
+ add_libc_test(
+ malloc_test
+ SUITE
+ libc-stdlib-tests
+ SRCS
+ block_test.cpp
+ malloc_test.cpp
+ freelist_malloc_test.cpp
+ freelist_heap_test.cpp
+ freelist_test.cpp
+ DEPENDS
+ libc.include.stdlib
+ libc.src.string.memcmp
+ libc.src.stdlib.malloc
+ libc.src.stdlib.free
+ )
endif()
diff --git a/libc/test/src/stdlib/freelist_heap_test.cpp b/libc/test/src/stdlib/freelist_heap_test.cpp
new file mode 100644
index 0000000000000..73419d25c6e37
--- /dev/null
+++ b/libc/test/src/stdlib/freelist_heap_test.cpp
@@ -0,0 +1,241 @@
+//===-- Unittests for freelist_heap ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/span.h"
+#include "src/stdlib/freelist_heap.h"
+#include "src/string/memcmp.h"
+#include "src/string/memcpy.h"
+#include "test/UnitTest/Test.h"
+
+namespace LIBC_NAMESPACE {
+
+TEST(LlvmLibcFreeListHeap, CanAllocate) {
+ constexpr size_t N = 2048;
+ constexpr size_t kAllocSize = 512;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
+
+ FreeListHeap<> allocator(buf);
+
+ void *ptr = allocator.Allocate(kAllocSize);
+
+ ASSERT_NE(ptr, static_cast<void *>(nullptr));
+ // In this case, the allocator should be returning us the start of the chunk.
+ EXPECT_EQ(ptr, static_cast<void *>(
+ &buf[0] + FreeListHeap<>::BlockType::BLOCK_OVERHEAD));
+}
+
+TEST(LlvmLibcFreeListHeap, AllocationsDontOverlap) {
+ constexpr size_t N = 2048;
+ constexpr size_t kAllocSize = 512;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
+
+ FreeListHeap<> allocator(buf);
+
+ void *ptr1 = allocator.Allocate(kAllocSize);
+ void *ptr2 = allocator.Allocate(kAllocSize);
+
+ ASSERT_NE(ptr1, static_cast<void *>(nullptr));
+ ASSERT_NE(ptr2, static_cast<void *>(nullptr));
+
+ uintptr_t ptr1_start = reinterpret_cast<uintptr_t>(ptr1);
+ uintptr_t ptr1_end = ptr1_start + kAllocSize;
+ uintptr_t ptr2_start = reinterpret_cast<uintptr_t>(ptr2);
+
+ EXPECT_GT(ptr2_start, ptr1_end);
+}
+
+TEST(LlvmLibcFreeListHeap, CanFreeAndRealloc) {
+ // There's not really a nice way to test that Free works, apart from to try
+ // and get that value back again.
+ constexpr size_t N = 2048;
+ constexpr size_t kAllocSize = 512;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
+
+ FreeListHeap<> allocator(buf);
+
+ void *ptr1 = allocator.Allocate(kAllocSize);
+ allocator.Free(ptr1);
+ void *ptr2 = allocator.Allocate(kAllocSize);
+
+ EXPECT_EQ(ptr1, ptr2);
+}
+
+TEST(LlvmLibcFreeListHeap, ReturnsNullWhenAllocationTooLarge) {
+ constexpr size_t N = 2048;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
+
+ FreeListHeap<> allocator(buf);
+
+ EXPECT_EQ(allocator.Allocate(N), static_cast<void *>(nullptr));
+}
+
+TEST(LlvmLibcFreeListHeap, ReturnsNullWhenFull) {
+ constexpr size_t N = 2048;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
+
+ FreeListHeap<> allocator(buf);
+
+ EXPECT_NE(allocator.Allocate(N - FreeListHeap<>::BlockType::BLOCK_OVERHEAD),
+ static_cast<void *>(nullptr));
+ EXPECT_EQ(allocator.Allocate(1), static_cast<void *>(nullptr));
+}
+
+TEST(LlvmLibcFreeListHeap, ReturnedPointersAreAligned) {
+ constexpr size_t N = 2048;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
+
+ FreeListHeap<> allocator(buf);
+
+ void *ptr1 = allocator.Allocate(1);
+
+ // Should be aligned to native pointer alignment
+ uintptr_t ptr1_start = reinterpret_cast<uintptr_t>(ptr1);
+ size_t alignment = alignof(void *);
+
+ EXPECT_EQ(ptr1_start % alignment, static_cast<size_t>(0));
+
+ void *ptr2 = allocator.Allocate(1);
+ uintptr_t ptr2_start = reinterpret_cast<uintptr_t>(ptr2);
+
+ EXPECT_EQ(ptr2_start % alignment, static_cast<size_t>(0));
+}
+
+TEST(LlvmLibcFreeListHeap, CanRealloc) {
+ constexpr size_t N = 2048;
+ constexpr size_t kAllocSize = 512;
+ constexpr size_t kNewAllocSize = 768;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(1)};
+
+ FreeListHeap<> allocator(buf);
+
+ void *ptr1 = allocator.Allocate(kAllocSize);
+ void *ptr2 = allocator.Realloc(ptr1, kNewAllocSize);
+
+ ASSERT_NE(ptr1, static_cast<void *>(nullptr));
+ ASSERT_NE(ptr2, static_cast<void *>(nullptr));
+}
+
+TEST(LlvmLibcFreeListHeap, ReallocHasSameContent) {
+ constexpr size_t N = 2048;
+ constexpr size_t kAllocSize = sizeof(int);
+ constexpr size_t kNewAllocSize = sizeof(int) * 2;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(1)};
+ // Data inside the allocated block.
+ cpp::byte data1[kAllocSize];
+ // Data inside the reallocated block.
+ cpp::byte data2[kAllocSize];
+
+ FreeListHeap<> allocator(buf);
+
+ int *ptr1 = reinterpret_cast<int *>(allocator.Allocate(kAllocSize));
+ *ptr1 = 42;
+ memcpy(data1, ptr1, kAllocSize);
+ int *ptr2 = reinterpret_cast<int *>(allocator.Realloc(ptr1, kNewAllocSize));
+ memcpy(data2, ptr2, kAllocSize);
+
+ ASSERT_NE(ptr1, static_cast<int *>(nullptr));
+ ASSERT_NE(ptr2, static_cast<int *>(nullptr));
+ // Verify that data inside the allocated and reallocated chunks are the same.
+ EXPECT_EQ(LIBC_NAMESPACE::memcmp(data1, data2, kAllocSize), 0);
+}
+
+TEST(LlvmLibcFreeListHeap, ReturnsNullReallocFreedPointer) {
+ constexpr size_t N = 2048;
+ constexpr size_t kAllocSize = 512;
+ constexpr size_t kNewAllocSize = 256;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
+
+ FreeListHeap<> allocator(buf);
+
+ void *ptr1 = allocator.Allocate(kAllocSize);
+ allocator.Free(ptr1);
+ void *ptr2 = allocator.Realloc(ptr1, kNewAllocSize);
+
+ EXPECT_EQ(static_cast<void *>(nullptr), ptr2);
+}
+
+TEST(LlvmLibcFreeListHeap, ReallocSmallerSize) {
+ constexpr size_t N = 2048;
+ constexpr size_t kAllocSize = 512;
+ constexpr size_t kNewAllocSize = 256;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
+
+ FreeListHeap<> allocator(buf);
+
+ void *ptr1 = allocator.Allocate(kAllocSize);
+ void *ptr2 = allocator.Realloc(ptr1, kNewAllocSize);
+
+ // For smaller sizes, Realloc will not shrink the block.
+ EXPECT_EQ(ptr1, ptr2);
+}
+
+TEST(LlvmLibcFreeListHeap, ReallocTooLarge) {
+ constexpr size_t N = 2048;
+ constexpr size_t kAllocSize = 512;
+ constexpr size_t kNewAllocSize = 4096;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
+
+ FreeListHeap<> allocator(buf);
+
+ void *ptr1 = allocator.Allocate(kAllocSize);
+ void *ptr2 = allocator.Realloc(ptr1, kNewAllocSize);
+
+ // Realloc() will not invalidate the original pointer if Reallc() fails
+ EXPECT_NE(static_cast<void *>(nullptr), ptr1);
+ EXPECT_EQ(static_cast<void *>(nullptr), ptr2);
+}
+
+TEST(LlvmLibcFreeListHeap, CanCalloc) {
+ constexpr size_t N = 2048;
+ constexpr size_t kAllocSize = 128;
+ constexpr size_t kNum = 4;
+ constexpr int size = kNum * kAllocSize;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(1)};
+ constexpr cpp::byte zero{0};
+
+ FreeListHeap<> allocator(buf);
+
+ cpp::byte *ptr1 =
+ reinterpret_cast<cpp::byte *>(allocator.Calloc(kNum, kAllocSize));
+
+ // Calloc'd content is zero.
+ for (int i = 0; i < size; i++) {
+ EXPECT_EQ(ptr1[i], zero);
+ }
+}
+
+TEST(LlvmLibcFreeListHeap, CanCallocWeirdSize) {
+ constexpr size_t N = 2048;
+ constexpr size_t kAllocSize = 143;
+ constexpr size_t kNum = 3;
+ constexpr int size = kNum * kAllocSize;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(132)};
+ constexpr cpp::byte zero{0};
+
+ FreeListHeap<> allocator(buf);
+
+ cpp::byte *ptr1 =
+ reinterpret_cast<cpp::byte *>(allocator.Calloc(kNum, kAllocSize));
+
+ // Calloc'd content is zero.
+ for (int i = 0; i < size; i++) {
+ EXPECT_EQ(ptr1[i], zero);
+ }
+}
+
+TEST(LlvmLibcFreeListHeap, CallocTooLarge) {
+ constexpr size_t N = 2048;
+ constexpr size_t kAllocSize = 2049;
+ alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(1)};
+
+ FreeListHeap<> allocator(buf);
+
+ EXPECT_EQ(allocator.Calloc(1, kAllocSize), static_cast<void *>(nullptr));
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/test/src/stdlib/freelist_malloc_test.cpp b/libc/test/src/stdlib/freelist_malloc_test.cpp
new file mode 100644
index 0000000000000..4844ed0127a32
--- /dev/null
+++ b/libc/test/src/stdlib/freelist_malloc_test.cpp
@@ -0,0 +1,56 @@
+//===-- Unittests for freelist_malloc -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdlib/calloc.h"
+#include "src/stdlib/free.h"
+#include "src/stdlib/freelist_heap.h"
+#include "src/stdlib/malloc.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::freelist_heap;
+
+TEST(LlvmLibcFreeListMalloc, ReplacingMalloc) {
+ constexpr size_t kAllocSize = 256;
+ constexpr size_t kCallocNum = 4;
+ constexpr size_t kCallocSize = 64;
+
+ uint8_t kBuff[4096];
+ LIBC_NAMESPACE::MallocInit(kBuff, kBuff + sizeof(kBuff));
+
+ void *ptr1 = LIBC_NAMESPACE::malloc(kAllocSize);
+
+ const auto &freelist_heap_stats = freelist_heap->heap_stats();
+
+ ASSERT_NE(ptr1, static_cast<void *>(nullptr));
+ EXPECT_EQ(freelist_heap_stats.bytes_allocated, kAllocSize);
+ EXPECT_EQ(freelist_heap_stats.cumulative_allocated, kAllocSize);
+ EXPECT_EQ(freelist_heap_stats.cumulative_freed, size_t(0));
+
+ LIBC_NAMESPACE::free(ptr1);
+ EXPECT_EQ(freelist_heap_stats.bytes_allocated, size_t(0));
+ EXPECT_EQ(freelist_heap_stats.cumulative_allocated, kAllocSize);
+ EXPECT_EQ(freelist_heap_stats.cumulative_freed, kAllocSize);
+
+ void *ptr2 = LIBC_NAMESPACE::calloc(kCallocNum, kCallocSize);
+ ASSERT_NE(ptr2, static_cast<void *>(nullptr));
+ EXPECT_EQ(freelist_heap_stats.bytes_allocated, kCallocNum * kCallocSize);
+ EXPECT_EQ(freelist_heap_stats.cumulative_allocated,
+ kAllocSize + kCallocNum * kCallocSize);
+ EXPECT_EQ(freelist_heap_stats.cumulative_freed, kAllocSize);
+
+ for (size_t i = 0; i < kCallocNum * kCallocSize; ++i) {
+ EXPECT_EQ(reinterpret_cast<uint8_t *>(ptr2)[i], uint8_t(0));
+ }
+
+ LIBC_NAMESPACE::free(ptr2);
+ EXPECT_EQ(freelist_heap_stats.bytes_allocated, size_t(0));
+ EXPECT_EQ(freelist_heap_stats.cumulative_allocated,
+ kAllocSize + kCallocNum * kCallocSize);
+ EXPECT_EQ(freelist_heap_stats.cumulative_freed,
+ kAllocSize + kCallocNum * kCallocSize);
+}
diff --git a/libc/test/src/stdlib/freelist_test.cpp b/libc/test/src/stdlib/freelist_test.cpp
new file mode 100644
index 0000000000000..f193624572c72
--- /dev/null
+++ b/libc/test/src/stdlib/freelist_test.cpp
@@ -0,0 +1,166 @@
+//===-- Unittests for a freelist --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <stddef.h>
+
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/span.h"
+#include "src/stdlib/freelist.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::FreeList;
+using LIBC_NAMESPACE::cpp::array;
+using LIBC_NAMESPACE::cpp::byte;
+using LIBC_NAMESPACE::cpp::span;
+
+static constexpr size_t SIZE = 8;
+static constexpr array<size_t, SIZE> example_sizes = {64, 128, 256, 512,
+ 1024, 2048, 4096, 8192};
+
+TEST(LlvmLibcFreeList, EmptyListHasNoMembers) {
+ FreeList<SIZE> list(example_sizes);
+
+ auto item = list.FindChunk(4);
+ EXPECT_EQ(item.size(), static_cast<size_t>(0));
+ item = list.FindChunk(128);
+ EXPECT_EQ(item.size(), static_cast<size_t>(0));
+}
+
+TEST(LlvmLibcFreeList, CanRetrieveAddedMember) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t kN = 512;
+
+ byte data[kN] = {byte(0)};
+
+ bool ok = list.AddChunk(span<byte>(data, kN));
+ EXPECT_TRUE(ok);
+
+ auto item = list.FindChunk(kN);
+ EXPECT_EQ(item.size(), kN);
+ EXPECT_EQ(item.data(), data);
+}
+
+TEST(LlvmLibcFreeList, CanRetrieveAddedMemberForSmallerSize) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t kN = 512;
+
+ byte data[kN] = {byte(0)};
+
+ ASSERT_TRUE(list.AddChunk(span<byte>(data, kN)));
+ auto item = list.FindChunk(kN / 2);
+ EXPECT_EQ(item.size(), kN);
+ EXPECT_EQ(item.data(), data);
+}
+
+TEST(LlvmLibcFreeList, CanRemoveItem) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t kN = 512;
+
+ byte data[kN] = {byte(0)};
+
+ ASSERT_TRUE(list.AddChunk(span<byte>(data, kN)));
+ EXPECT_TRUE(list.RemoveChunk(span<byte>(data, kN)));
+
+ auto item = list.FindChunk(kN);
+ EXPECT_EQ(item.size(), static_cast<size_t>(0));
+}
+
+TEST(LlvmLibcFreeList, FindReturnsSmallestChunk) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t kN1 = 512;
+ constexpr size_t kN2 = 1024;
+
+ byte data1[kN1] = {byte(0)};
+ byte data2[kN2] = {byte(0)};
+
+ ASSERT_TRUE(list.AddChunk(span<byte>(data1, kN1)));
+ ASSERT_TRUE(list.AddChunk(span<byte>(data2, kN2)));
+
+ auto chunk = list.FindChunk(kN1 / 2);
+ EXPECT_EQ(chunk.size(), kN1);
+ EXPECT_EQ(chunk.data(), data1);
+
+ chunk = list.FindChunk(kN1);
+ EXPECT_EQ(chunk.size(), kN1);
+ EXPECT_EQ(chunk.data(), data1);
+
+ chunk = list.FindChunk(kN1 + 1);
+ EXPECT_EQ(chunk.size(), kN2);
+ EXPECT_EQ(chunk.data(), data2);
+}
+
+TEST(LlvmLibcFreeList, FindReturnsCorrectChunkInSameBucket) {
+ // If we have two values in the same bucket, ensure that the allocation will
+ // pick an appropriately sized one.
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t kN1 = 512;
+ constexpr size_t kN2 = 257;
+
+ byte data1[kN1] = {byte(0)};
+ byte data2[kN2] = {byte(0)};
+
+ // List should now be 257 -> 512 -> NULL
+ ASSERT_TRUE(list.AddChunk(span<byte>(data1, kN1)));
+ ASSERT_TRUE(list.AddChunk(span<byte>(data2, kN2)));
+
+ auto chunk = list.FindChunk(kN2 + 1);
+ EXPECT_EQ(chunk.size(), kN1);
+}
+
+TEST(LlvmLibcFreeList, FindCanMoveUpThroughBuckets) {
+ // Ensure that finding a chunk will move up through buckets if no appropriate
+ // chunks were found in a given bucket
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t kN1 = 257;
+ constexpr size_t kN2 = 513;
+
+ byte data1[kN1] = {byte(0)};
+ byte data2[kN2] = {byte(0)};
+
+ // List should now be:
+ // bkt[3] (257 bytes up to 512 bytes) -> 257 -> NULL
+ // bkt[4] (513 bytes up to 1024 bytes) -> 513 -> NULL
+ ASSERT_TRUE(list.AddChunk(span<byte>(data1, kN1)));
+ ASSERT_TRUE(list.AddChunk(span<byte>(data2, kN2)));
+
+ // Request a 300 byte chunk. This should return the 513 byte one
+ auto chunk = list.FindChunk(kN1 + 1);
+ EXPECT_EQ(chunk.size(), kN2);
+}
+
+TEST(LlvmLibcFreeList, RemoveUnknownChunkReturnsNotFound) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t kN = 512;
+
+ byte data[kN] = {byte(0)};
+ byte data2[kN] = {byte(0)};
+
+ ASSERT_TRUE(list.AddChunk(span<byte>(data, kN)));
+ EXPECT_FALSE(list.RemoveChunk(span<byte>(data2, kN)));
+}
+
+TEST(LlvmLibcFreeList, CanStoreMultipleChunksPerBucket) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t kN = 512;
+
+ byte data1[kN] = {byte(0)};
+ byte data2[kN] = {byte(0)};
+
+ ASSERT_TRUE(list.AddChunk(span<byte>(data1, kN)));
+ ASSERT_TRUE(list.AddChunk(span<byte>(data2, kN)));
+
+ auto chunk1 = list.FindChunk(kN);
+ ASSERT_TRUE(list.RemoveChunk(chunk1));
+ auto chunk2 = list.FindChunk(kN);
+ ASSERT_TRUE(list.RemoveChunk(chunk2));
+
+ // Ordering of the chunks doesn't matter
+ EXPECT_TRUE(chunk1.data() != chunk2.data());
+ EXPECT_TRUE(chunk1.data() == data1 || chunk1.data() == data2);
+ EXPECT_TRUE(chunk2.data() == data1 || chunk2.data() == data2);
+}
diff --git a/libc/test/src/stdlib/malloc_test.cpp b/libc/test/src/stdlib/malloc_test.cpp
index d9023cf56d9fe..6fb07bec9f336 100644
--- a/libc/test/src/stdlib/malloc_test.cpp
+++ b/libc/test/src/stdlib/malloc_test.cpp
@@ -7,10 +7,14 @@
//===----------------------------------------------------------------------===//
#include "src/stdlib/free.h"
+#include "src/stdlib/freelist_heap.h"
#include "src/stdlib/malloc.h"
#include "test/UnitTest/Test.h"
TEST(LlvmLibcMallocTest, Allocate) {
+ uint8_t kBuff[1024];
+ LIBC_NAMESPACE::MallocInit(kBuff, kBuff + sizeof(kBuff));
+
int *ptr = reinterpret_cast<int *>(LIBC_NAMESPACE::malloc(sizeof(int)));
EXPECT_NE(reinterpret_cast<void *>(ptr), static_cast<void *>(nullptr));
*ptr = 1;
More information about the libc-commits
mailing list