[libc-commits] [libc] [WIP][libc] Use best-fit binary trie to make malloc logarithmic (PR #106259)
Daniel Thornburgh via libc-commits
libc-commits at lists.llvm.org
Thu Sep 5 13:23:58 PDT 2024
https://github.com/mysterymath updated https://github.com/llvm/llvm-project/pull/106259
>From b1cfdc78401556e7aa21e9f7e3e209665386b2b8 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Mon, 5 Aug 2024 15:18:53 -0700
Subject: [PATCH 01/54] Write an alternate freelist
---
libc/src/__support/freelist2.h | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
create mode 100644 libc/src/__support/freelist2.h
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
new file mode 100644
index 00000000000000..5036458d91e2ce
--- /dev/null
+++ b/libc/src/__support/freelist2.h
@@ -0,0 +1,27 @@
+//===-- Interface for freelist --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST2_H
+#define LLVM_LIBC_SRC___SUPPORT_FREELIST2_H
+
+namespace LIBC_NAMESPACE_DECL {
+
+class FreeList {
+public:
+ class Node {
+ };
+
+private:
+ Node *begin_;
+};
+
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FREELIST2_H
>From 844803b6a664d003b0de34c23838a3a7ce0ef276 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 6 Aug 2024 11:06:17 -0700
Subject: [PATCH 02/54] Add simple unit test for freelist2
---
libc/src/__support/freelist2.h | 11 +-
libc/test/src/__support/CMakeLists.txt | 2 +
libc/test/src/__support/freelist2_test.cpp | 164 +++++++++++++++++++++
3 files changed, 175 insertions(+), 2 deletions(-)
create mode 100644 libc/test/src/__support/freelist2_test.cpp
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index 5036458d91e2ce..8fcfbe937c08e4 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -9,17 +9,24 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST2_H
#define LLVM_LIBC_SRC___SUPPORT_FREELIST2_H
+#include "block.h"
+
namespace LIBC_NAMESPACE_DECL {
-class FreeList {
+class FreeList2 {
public:
class Node {
};
+ bool empty() const { return !begin_; }
+
+ void push(Block<> *block);
+
private:
- Node *begin_;
+ Node *begin_ = nullptr;
};
+LIBC_INLINE void FreeList2::push(Block<> *block) {
}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 90de520405981b..7ff67aab085c58 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -21,9 +21,11 @@ if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
libc-support-tests
SRCS
freelist_test.cpp
+ freelist2_test.cpp
DEPENDS
libc.src.__support.CPP.array
libc.src.__support.CPP.span
+ libc.src.__support.block
libc.src.__support.freelist
)
endif()
diff --git a/libc/test/src/__support/freelist2_test.cpp b/libc/test/src/__support/freelist2_test.cpp
new file mode 100644
index 00000000000000..a304d0987fa34b
--- /dev/null
+++ b/libc/test/src/__support/freelist2_test.cpp
@@ -0,0 +1,164 @@
+//===-- Unittests for a freelist --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <stddef.h>
+
+#include "src/__support/freelist2.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::FreeList2;
+
+TEST(LlvmLibcFreeList2, DefaultListIsEmpty) {
+ FreeList2 list;
+ EXPECT_TRUE(list.empty());
+}
+
+#if 0
+TEST(LlvmLibcFreeList2, EmptyListHasNoMembers) {
+ FreeList<SIZE> list(example_sizes);
+
+ auto item = list.find_chunk(4);
+ EXPECT_EQ(item.size(), static_cast<size_t>(0));
+ item = list.find_chunk(128);
+ EXPECT_EQ(item.size(), static_cast<size_t>(0));
+}
+
+TEST(LlvmLibcFreeList, CanRetrieveAddedMember) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t N = 512;
+
+ byte data[N] = {byte(0)};
+
+ bool ok = list.add_chunk(span<byte>(data, N));
+ EXPECT_TRUE(ok);
+
+ auto item = list.find_chunk(N);
+ EXPECT_EQ(item.size(), N);
+ EXPECT_EQ(item.data(), data);
+}
+
+TEST(LlvmLibcFreeList, CanRetrieveAddedMemberForSmallerSize) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t N = 512;
+
+ byte data[N] = {byte(0)};
+
+ ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
+ auto item = list.find_chunk(N / 2);
+ EXPECT_EQ(item.size(), N);
+ EXPECT_EQ(item.data(), data);
+}
+
+TEST(LlvmLibcFreeList, CanRemoveItem) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t N = 512;
+
+ byte data[N] = {byte(0)};
+
+ ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
+ EXPECT_TRUE(list.remove_chunk(span<byte>(data, N)));
+
+ auto item = list.find_chunk(N);
+ EXPECT_EQ(item.size(), static_cast<size_t>(0));
+}
+
+TEST(LlvmLibcFreeList, FindReturnsSmallestChunk) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t kN1 = 512;
+ constexpr size_t kN2 = 1024;
+
+ byte data1[kN1] = {byte(0)};
+ byte data2[kN2] = {byte(0)};
+
+ ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
+ ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));
+
+ auto chunk = list.find_chunk(kN1 / 2);
+ EXPECT_EQ(chunk.size(), kN1);
+ EXPECT_EQ(chunk.data(), data1);
+
+ chunk = list.find_chunk(kN1);
+ EXPECT_EQ(chunk.size(), kN1);
+ EXPECT_EQ(chunk.data(), data1);
+
+ chunk = list.find_chunk(kN1 + 1);
+ EXPECT_EQ(chunk.size(), kN2);
+ EXPECT_EQ(chunk.data(), data2);
+}
+
+TEST(LlvmLibcFreeList, FindReturnsCorrectChunkInSameBucket) {
+ // If we have two values in the same bucket, ensure that the allocation will
+ // pick an appropriately sized one.
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t kN1 = 512;
+ constexpr size_t kN2 = 257;
+
+ byte data1[kN1] = {byte(0)};
+ byte data2[kN2] = {byte(0)};
+
+ // List should now be 257 -> 512 -> NULL
+ ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
+ ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));
+
+ auto chunk = list.find_chunk(kN2 + 1);
+ EXPECT_EQ(chunk.size(), kN1);
+}
+
+TEST(LlvmLibcFreeList, FindCanMoveUpThroughBuckets) {
+ // Ensure that finding a chunk will move up through buckets if no appropriate
+ // chunks were found in a given bucket
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t kN1 = 257;
+ constexpr size_t kN2 = 513;
+
+ byte data1[kN1] = {byte(0)};
+ byte data2[kN2] = {byte(0)};
+
+ // List should now be:
+ // bkt[3] (257 bytes up to 512 bytes) -> 257 -> NULL
+ // bkt[4] (513 bytes up to 1024 bytes) -> 513 -> NULL
+ ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
+ ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));
+
+ // Request a 300 byte chunk. This should return the 513 byte one
+ auto chunk = list.find_chunk(kN1 + 1);
+ EXPECT_EQ(chunk.size(), kN2);
+}
+
+TEST(LlvmLibcFreeList, RemoveUnknownChunkReturnsNotFound) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t N = 512;
+
+ byte data[N] = {byte(0)};
+ byte data2[N] = {byte(0)};
+
+ ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
+ EXPECT_FALSE(list.remove_chunk(span<byte>(data2, N)));
+}
+
+TEST(LlvmLibcFreeList, CanStoreMultipleChunksPerBucket) {
+ FreeList<SIZE> list(example_sizes);
+ constexpr size_t N = 512;
+
+ byte data1[N] = {byte(0)};
+ byte data2[N] = {byte(0)};
+
+ ASSERT_TRUE(list.add_chunk(span<byte>(data1, N)));
+ ASSERT_TRUE(list.add_chunk(span<byte>(data2, N)));
+
+ auto chunk1 = list.find_chunk(N);
+ ASSERT_TRUE(list.remove_chunk(chunk1));
+ auto chunk2 = list.find_chunk(N);
+ ASSERT_TRUE(list.remove_chunk(chunk2));
+
+ // Ordering of the chunks doesn't matter
+ EXPECT_TRUE(chunk1.data() != chunk2.data());
+ EXPECT_TRUE(chunk1.data() == data1 || chunk1.data() == data2);
+ EXPECT_TRUE(chunk2.data() == data1 || chunk2.data() == data2);
+}
+#endif
>From 6297dbd00c06a7afc8baf7cea258a2a4b221afe2 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 6 Aug 2024 12:00:59 -0700
Subject: [PATCH 03/54] Push to empty list
---
libc/src/__support/freelist2.h | 3 +++
libc/test/src/__support/freelist2_test.cpp | 16 +++++++++++++++-
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index 8fcfbe937c08e4..a58ec89d692c5f 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -27,6 +27,9 @@ class FreeList2 {
};
LIBC_INLINE void FreeList2::push(Block<> *block) {
+ Node *node = new (block->usable_space()) Node;
+ LIBC_ASSERT(!begin_ && "not yet implemented");
+ begin_ = node;
}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/freelist2_test.cpp b/libc/test/src/__support/freelist2_test.cpp
index a304d0987fa34b..074975d7cec567 100644
--- a/libc/test/src/__support/freelist2_test.cpp
+++ b/libc/test/src/__support/freelist2_test.cpp
@@ -8,16 +8,28 @@
#include <stddef.h>
+#include "src/__support/freelist2.h"
#include "src/__support/freelist2.h"
#include "test/UnitTest/Test.h"
-using LIBC_NAMESPACE::FreeList2;
+namespace LIBC_NAMESPACE_DECL {
TEST(LlvmLibcFreeList2, DefaultListIsEmpty) {
FreeList2 list;
EXPECT_TRUE(list.empty());
}
+TEST(LlvmLibcFreeList2, PushMakesListNonEmpty) {
+ cpp::byte mem[1024];
+ optional<Block<>*> maybeBlock = Block<>::init(mem);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<>* block = *maybeBlock;
+
+ FreeList2 list;
+ list.push(block);
+ EXPECT_FALSE(list.empty());
+}
+
#if 0
TEST(LlvmLibcFreeList2, EmptyListHasNoMembers) {
FreeList<SIZE> list(example_sizes);
@@ -162,3 +174,5 @@ TEST(LlvmLibcFreeList, CanStoreMultipleChunksPerBucket) {
EXPECT_TRUE(chunk2.data() == data1 || chunk2.data() == data2);
}
#endif
+
+} // namespace LIBC_NAMESPACE_DECL
>From 4d3e3c34bae5dc0eb6271bae806983c535c2a1ee Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 6 Aug 2024 12:03:28 -0700
Subject: [PATCH 04/54] Trivial pop
---
libc/src/__support/freelist2.h | 6 ++++++
libc/test/src/__support/freelist2_test.cpp | 12 ++++++++++++
2 files changed, 18 insertions(+)
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index a58ec89d692c5f..24a9425267cbc0 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -21,6 +21,7 @@ class FreeList2 {
bool empty() const { return !begin_; }
void push(Block<> *block);
+ void pop();
private:
Node *begin_ = nullptr;
@@ -32,6 +33,11 @@ LIBC_INLINE void FreeList2::push(Block<> *block) {
begin_ = node;
}
+LIBC_INLINE void FreeList2::pop() {
+ LIBC_ASSERT(!empty());
+ begin_ = nullptr;
+}
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREELIST2_H
diff --git a/libc/test/src/__support/freelist2_test.cpp b/libc/test/src/__support/freelist2_test.cpp
index 074975d7cec567..f31eaefe42e4a6 100644
--- a/libc/test/src/__support/freelist2_test.cpp
+++ b/libc/test/src/__support/freelist2_test.cpp
@@ -30,6 +30,18 @@ TEST(LlvmLibcFreeList2, PushMakesListNonEmpty) {
EXPECT_FALSE(list.empty());
}
+TEST(LlvmLibcFreeList2, PushPopEmptiesList) {
+ cpp::byte mem[1024];
+ optional<Block<>*> maybeBlock = Block<>::init(mem);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<>* block = *maybeBlock;
+
+ FreeList2 list;
+ list.push(block);
+ list.pop();
+ EXPECT_TRUE(list.empty());
+}
+
#if 0
TEST(LlvmLibcFreeList2, EmptyListHasNoMembers) {
FreeList<SIZE> list(example_sizes);
>From 9384969cd80d4374cf27ce05963f63941663e5af Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 6 Aug 2024 13:58:55 -0700
Subject: [PATCH 05/54] Add front test
---
libc/src/__support/freelist2.h | 11 ++++++++---
libc/test/src/__support/freelist2_test.cpp | 23 +++++++---------------
2 files changed, 15 insertions(+), 19 deletions(-)
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index 24a9425267cbc0..70ba2dcd4b168c 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -15,10 +15,10 @@ namespace LIBC_NAMESPACE_DECL {
class FreeList2 {
public:
- class Node {
- };
+ class Node {};
bool empty() const { return !begin_; }
+ const Node &front() const;
void push(Block<> *block);
void pop();
@@ -27,6 +27,11 @@ class FreeList2 {
Node *begin_ = nullptr;
};
+const FreeList2::Node &FreeList2::front() const {
+ LIBC_ASSERT(!empty());
+ return *begin_;
+}
+
LIBC_INLINE void FreeList2::push(Block<> *block) {
Node *node = new (block->usable_space()) Node;
LIBC_ASSERT(!begin_ && "not yet implemented");
@@ -39,5 +44,5 @@ LIBC_INLINE void FreeList2::pop() {
}
} // namespace LIBC_NAMESPACE_DECL
-
+
#endif // LLVM_LIBC_SRC___SUPPORT_FREELIST2_H
diff --git a/libc/test/src/__support/freelist2_test.cpp b/libc/test/src/__support/freelist2_test.cpp
index f31eaefe42e4a6..403c6e3db9518b 100644
--- a/libc/test/src/__support/freelist2_test.cpp
+++ b/libc/test/src/__support/freelist2_test.cpp
@@ -8,36 +8,27 @@
#include <stddef.h>
-#include "src/__support/freelist2.h"
#include "src/__support/freelist2.h"
#include "test/UnitTest/Test.h"
namespace LIBC_NAMESPACE_DECL {
-TEST(LlvmLibcFreeList2, DefaultListIsEmpty) {
+TEST(LlvmLibcFreeList2, Contruct) {
FreeList2 list;
EXPECT_TRUE(list.empty());
}
-TEST(LlvmLibcFreeList2, PushMakesListNonEmpty) {
- cpp::byte mem[1024];
- optional<Block<>*> maybeBlock = Block<>::init(mem);
- ASSERT_TRUE(maybeBlock.has_value());
- Block<>* block = *maybeBlock;
-
- FreeList2 list;
- list.push(block);
- EXPECT_FALSE(list.empty());
-}
-
-TEST(LlvmLibcFreeList2, PushPopEmptiesList) {
+TEST(LlvmLibcFreeList2, PushPop) {
cpp::byte mem[1024];
- optional<Block<>*> maybeBlock = Block<>::init(mem);
+ optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
- Block<>* block = *maybeBlock;
+ Block<> *block = *maybeBlock;
FreeList2 list;
list.push(block);
+ ASSERT_FALSE(list.empty());
+ EXPECT_EQ(&list.front(),
+ reinterpret_cast<const FreeList2::Node *>(block->usable_space()));
list.pop();
EXPECT_TRUE(list.empty());
}
>From 64bf42af821b6c81d154d866ce5350683f79c38c Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 6 Aug 2024 14:05:02 -0700
Subject: [PATCH 06/54] Multiple entry push and pop (FIFO)
---
libc/src/__support/freelist2.h | 42 ++++++++++++++++++----
libc/test/src/__support/freelist2_test.cpp | 21 +++++++----
2 files changed, 50 insertions(+), 13 deletions(-)
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index 70ba2dcd4b168c..91118f0409e6d8 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -13,34 +13,62 @@
namespace LIBC_NAMESPACE_DECL {
+/// A FIFO free-list storing Blocks of the same size.
class FreeList2 {
+private:
+ // A circular doubly-linked node.
+ struct Node {
+ Node *prev;
+ Node *next;
+ };
+
public:
- class Node {};
+ // Blocks with inner sizes smaller than this must not be pushed.
+ static constexpr size_t MIN_INNER_SIZE = sizeof(Node);
bool empty() const { return !begin_; }
- const Node &front() const;
+ Block<> *front() const;
+ /// Push to the back.
void push(Block<> *block);
+
+ /// Pop the front.
void pop();
private:
Node *begin_ = nullptr;
};
-const FreeList2::Node &FreeList2::front() const {
+Block<> *FreeList2::front() const {
LIBC_ASSERT(!empty());
- return *begin_;
+ return Block<>::from_usable_space(begin_);
}
LIBC_INLINE void FreeList2::push(Block<> *block) {
+ LIBC_ASSERT(block->inner_size() >= MIN_INNER_SIZE &&
+ "block too small to accomodate free list node");
Node *node = new (block->usable_space()) Node;
- LIBC_ASSERT(!begin_ && "not yet implemented");
- begin_ = node;
+ if (begin_) {
+ LIBC_ASSERT(block->outer_size() == front()->outer_size() &&
+ "freelist entries must have the same size");
+ node->prev = begin_->prev;
+ node->next = begin_;
+ begin_->prev->next = node;
+ begin_->prev = node;
+ } else {
+ begin_ = node->prev = node->next = node;
+ }
}
LIBC_INLINE void FreeList2::pop() {
LIBC_ASSERT(!empty());
- begin_ = nullptr;
+ if (begin_->next == begin_) {
+ begin_ = nullptr;
+ } else {
+ begin_->prev->next = begin_->next;
+ begin_->next->prev = begin_->prev;
+ begin_ = begin_->next;
+ }
}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/freelist2_test.cpp b/libc/test/src/__support/freelist2_test.cpp
index 403c6e3db9518b..abac9b11b8153f 100644
--- a/libc/test/src/__support/freelist2_test.cpp
+++ b/libc/test/src/__support/freelist2_test.cpp
@@ -19,16 +19,25 @@ TEST(LlvmLibcFreeList2, Contruct) {
}
TEST(LlvmLibcFreeList2, PushPop) {
- cpp::byte mem[1024];
- optional<Block<> *> maybeBlock = Block<>::init(mem);
+ cpp::byte mem1[1024];
+ optional<Block<> *> maybeBlock = Block<>::init(mem1);
ASSERT_TRUE(maybeBlock.has_value());
- Block<> *block = *maybeBlock;
+ Block<> *block1 = *maybeBlock;
+
+ cpp::byte mem2[1024];
+ maybeBlock = Block<>::init(mem2);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *block2 = *maybeBlock;
FreeList2 list;
- list.push(block);
+ list.push(block1);
+ ASSERT_FALSE(list.empty());
+ EXPECT_EQ(list.front(), block1);
+ list.push(block2);
+ EXPECT_EQ(list.front(), block1);
+ list.pop();
ASSERT_FALSE(list.empty());
- EXPECT_EQ(&list.front(),
- reinterpret_cast<const FreeList2::Node *>(block->usable_space()));
+ EXPECT_EQ(list.front(), block2);
list.pop();
EXPECT_TRUE(list.empty());
}
>From 1926af8d3a98f3f2de765c0bef074b4f9f37279b Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 6 Aug 2024 14:55:00 -0700
Subject: [PATCH 07/54] Sketch a freetrie class
---
libc/src/__support/freelist2.h | 4 +-
libc/src/__support/freetrie.h | 77 +++++++++++++++++++++++
libc/test/src/__support/CMakeLists.txt | 1 +
libc/test/src/__support/freetrie_test.cpp | 20 ++++++
4 files changed, 100 insertions(+), 2 deletions(-)
create mode 100644 libc/src/__support/freetrie.h
create mode 100644 libc/test/src/__support/freetrie_test.cpp
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index 91118f0409e6d8..f3baaebc8a6d95 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -15,7 +15,7 @@ namespace LIBC_NAMESPACE_DECL {
/// A FIFO free-list storing Blocks of the same size.
class FreeList2 {
-private:
+protected:
// A circular doubly-linked node.
struct Node {
Node *prev;
@@ -39,7 +39,7 @@ class FreeList2 {
Node *begin_ = nullptr;
};
-Block<> *FreeList2::front() const {
+LIBC_INLINE Block<> *FreeList2::front() const {
LIBC_ASSERT(!empty());
return Block<>::from_usable_space(begin_);
}
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
new file mode 100644
index 00000000000000..45341ef668eed4
--- /dev/null
+++ b/libc/src/__support/freetrie.h
@@ -0,0 +1,77 @@
+//===-- Interface for freetrie --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
+#define LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
+
+#include "freelist2.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+/// A trie representing a map of free lists covering a contiguous SizeRange.
+class FreeTrie : public FreeList2 {
+private:
+ // A subtrie of free lists covering a contiguous SizeRange. This is also a
+ // free list with a size somewhere within the range. There is no relationship
+ // between the size of this free list and the sizes of the lower and upper
+ // subtries.
+ struct Node : public FreeList2::Node {
+ // The containing trie or nullptr if this is the root.
+ Node *parent;
+ // The child subtrie covering the lower half of this subtrie's size range.
+ Node *lower;
+ // The child subtrie covering the upper half of this subtrie's size range.
+ Node *upper;
+ };
+
+public:
+ // Blocks with inner sizes smaller than this must not be pushed.
+ static constexpr size_t MIN_INNER_SIZE = sizeof(Node);
+
+ // Power-of-two range of sizes covered by a subtrie.
+ class SizeRange {
+ public:
+ SizeRange(size_t min, size_t width);
+
+ /// Return the lower half of the size range.
+ SizeRange lower() const;
+
+ /// Return the lower half of the size range.
+ SizeRange upper() const;
+
+ /// Return the split point between lower and upper.
+ size_t middle() const;
+
+ private:
+ size_t min;
+ size_t width;
+ };
+
+ /// Push to the correctly-sized freelist. The caller must provide the size
+ /// range for this trie, since it isn't stored within.
+ void push(Block<> *block, SizeRange range);
+};
+
+LIBC_INLINE FreeTrie::SizeRange::SizeRange(size_t min, size_t width)
+ : min(min), width(width) {
+ LIBC_ASSERT(!(width & (width - 1)) && "width must be a power of two");
+}
+
+LIBC_INLINE FreeTrie::SizeRange FreeTrie::SizeRange::lower() const {
+ return {min, width / 2};
+}
+LIBC_INLINE FreeTrie::SizeRange FreeTrie::SizeRange::upper() const {
+ return {middle(), width / 2};
+}
+LIBC_INLINE size_t FreeTrie::SizeRange::middle() const {
+ return min + width / 2;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 7ff67aab085c58..e012e3345afde4 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -22,6 +22,7 @@ if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
SRCS
freelist_test.cpp
freelist2_test.cpp
+ freetrie_test.cpp
DEPENDS
libc.src.__support.CPP.array
libc.src.__support.CPP.span
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
new file mode 100644
index 00000000000000..3b6fbb8fb6e530
--- /dev/null
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -0,0 +1,20 @@
+//===-- Unittests for a freetrie --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <stddef.h>
+
+#include "src/__support/freetrie.h"
+#include "test/UnitTest/Test.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+TEST(LlvmLibcFreeTrie, Null) {
+ EXPECT_TRUE(true);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
>From 88eb78c5d1599cca0f79a209438a8679e7b503a9 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 6 Aug 2024 16:42:47 -0700
Subject: [PATCH 08/54] Implement raw push
---
libc/src/__support/freelist2.h | 30 +++++++++++++++++++-----------
libc/src/__support/freetrie.h | 15 ++++++++++++---
2 files changed, 31 insertions(+), 14 deletions(-)
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index f3baaebc8a6d95..15ed526ad169a5 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -35,6 +35,10 @@ class FreeList2 {
/// Pop the front.
void pop();
+protected:
+ /// Push an already-constructed node to the back.
+ void push(Node *node);
+
private:
Node *begin_ = nullptr;
};
@@ -47,17 +51,7 @@ LIBC_INLINE Block<> *FreeList2::front() const {
LIBC_INLINE void FreeList2::push(Block<> *block) {
LIBC_ASSERT(block->inner_size() >= MIN_INNER_SIZE &&
"block too small to accomodate free list node");
- Node *node = new (block->usable_space()) Node;
- if (begin_) {
- LIBC_ASSERT(block->outer_size() == front()->outer_size() &&
- "freelist entries must have the same size");
- node->prev = begin_->prev;
- node->next = begin_;
- begin_->prev->next = node;
- begin_->prev = node;
- } else {
- begin_ = node->prev = node->next = node;
- }
+ push(new (block->usable_space()) Node);
}
LIBC_INLINE void FreeList2::pop() {
@@ -71,6 +65,20 @@ LIBC_INLINE void FreeList2::pop() {
}
}
+LIBC_INLINE void FreeList2::push(Node *node) {
+ if (begin_) {
+ LIBC_ASSERT(Block<>::from_usable_space(node)->outer_size() ==
+ front()->outer_size() &&
+ "freelist entries must have the same size");
+ node->prev = begin_->prev;
+ node->next = begin_;
+ begin_->prev->next = node;
+ begin_->prev = node;
+ } else {
+ begin_ = node->prev = node->next = node;
+ }
+}
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREELIST2_H
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 45341ef668eed4..f1f61a3b8fd573 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -30,9 +30,6 @@ class FreeTrie : public FreeList2 {
};
public:
- // Blocks with inner sizes smaller than this must not be pushed.
- static constexpr size_t MIN_INNER_SIZE = sizeof(Node);
-
// Power-of-two range of sizes covered by a subtrie.
class SizeRange {
public:
@@ -52,6 +49,12 @@ class FreeTrie : public FreeList2 {
size_t width;
};
+ // Blocks with inner sizes smaller than this must not be pushed.
+ static constexpr size_t MIN_INNER_SIZE = sizeof(Node);
+
+ /// Push to this freelist.
+ void push(Block<> *block);
+
/// Push to the correctly-sized freelist. The caller must provide the size
/// range for this trie, since it isn't stored within.
void push(Block<> *block, SizeRange range);
@@ -72,6 +75,12 @@ LIBC_INLINE size_t FreeTrie::SizeRange::middle() const {
return min + width / 2;
}
+LIBC_INLINE void FreeTrie::push(Block<> *block) {
+ LIBC_ASSERT(block->inner_size() >= MIN_INNER_SIZE &&
+ "block too small to accomodate free list node");
+ FreeList2::push(new (block->usable_space()) Node);
+}
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
>From dbc186d83550bb3477396112fb3aeda0731220d6 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 7 Aug 2024 11:40:14 -0700
Subject: [PATCH 09/54] Add freetrie push tests
---
libc/src/__support/freetrie.h | 20 ++++++++++++++++----
libc/test/src/__support/freetrie_test.cpp | 17 +++++++++++++++--
2 files changed, 31 insertions(+), 6 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index f1f61a3b8fd573..c1878cfafb36cb 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -35,13 +35,13 @@ class FreeTrie : public FreeList2 {
public:
SizeRange(size_t min, size_t width);
- /// Return the lower half of the size range.
+ /// @returns The lower half of the size range.
SizeRange lower() const;
- /// Return the lower half of the size range.
+ /// @returns The lower half of the size range.
SizeRange upper() const;
- /// Return the split point between lower and upper.
+ /// @returns The split point between lower and upper.
size_t middle() const;
private:
@@ -78,7 +78,19 @@ LIBC_INLINE size_t FreeTrie::SizeRange::middle() const {
LIBC_INLINE void FreeTrie::push(Block<> *block) {
LIBC_ASSERT(block->inner_size() >= MIN_INNER_SIZE &&
"block too small to accomodate free list node");
- FreeList2::push(new (block->usable_space()) Node);
+ Node *node = new (block->usable_space()) Node;
+ if (empty())
+ node->parent = node->lower = node->upper = nullptr;
+ FreeList2::push(node);
+}
+
+LIBC_INLINE void FreeTrie::push(Block<> *block, SizeRange range) {
+ if (empty() || block->outer_size() == front()->outer_size()) {
+ push(block);
+ return;
+ }
+
+ // TODO;
}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 3b6fbb8fb6e530..7763b38b15b980 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -13,8 +13,21 @@
namespace LIBC_NAMESPACE_DECL {
-TEST(LlvmLibcFreeTrie, Null) {
- EXPECT_TRUE(true);
+TEST(LlvmLibcFreeTrie, Construct) {
+ FreeTrie trie;
+ EXPECT_TRUE(trie.empty());
+}
+
+TEST(LlvmLibcFreeTrie, Push) {
+ cpp::byte mem1[1024];
+ optional<Block<> *> maybeBlock = Block<>::init(mem1);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *block = *maybeBlock;
+
+ FreeTrie trie;
+ trie.push(block);
+ ASSERT_FALSE(trie.empty());
+ EXPECT_EQ(trie.front(), block);
}
} // namespace LIBC_NAMESPACE_DECL
>From 09fc08aa41e475d7cc3355ff023d4e063fa83603 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 7 Aug 2024 14:39:20 -0700
Subject: [PATCH 10/54] Some refactoring.
---
libc/test/src/__support/freetrie_test.cpp | 24 +++++++++++++++++------
1 file changed, 18 insertions(+), 6 deletions(-)
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 7763b38b15b980..fef5d16bb89ac6 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -18,16 +18,28 @@ TEST(LlvmLibcFreeTrie, Construct) {
EXPECT_TRUE(trie.empty());
}
-TEST(LlvmLibcFreeTrie, Push) {
+TEST(LlvmLibcFreeList2, PushPopDirect) {
cpp::byte mem1[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem1);
ASSERT_TRUE(maybeBlock.has_value());
- Block<> *block = *maybeBlock;
+ Block<> *block1 = *maybeBlock;
- FreeTrie trie;
- trie.push(block);
- ASSERT_FALSE(trie.empty());
- EXPECT_EQ(trie.front(), block);
+ cpp::byte mem2[1024];
+ maybeBlock = Block<>::init(mem2);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *block2 = *maybeBlock;
+
+ FreeList2 list;
+ list.push(block1);
+ ASSERT_FALSE(list.empty());
+ EXPECT_EQ(list.front(), block1);
+ list.push(block2);
+ EXPECT_EQ(list.front(), block1);
+ list.pop();
+ ASSERT_FALSE(list.empty());
+ EXPECT_EQ(list.front(), block2);
+ list.pop();
+ EXPECT_TRUE(list.empty());
}
} // namespace LIBC_NAMESPACE_DECL
>From b044353f2669a3f80d5fde189a4e4091c499c19d Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 7 Aug 2024 15:07:05 -0700
Subject: [PATCH 11/54] Recast freelist and freetrie as raw nodes
---
libc/src/__support/freelist2.h | 72 ++++-----
libc/src/__support/freetrie.h | 65 ++++----
libc/test/src/__support/freelist2_test.cpp | 172 ++-------------------
libc/test/src/__support/freetrie_test.cpp | 29 ++--
4 files changed, 85 insertions(+), 253 deletions(-)
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index 15ed526ad169a5..1bedc9de8424c7 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -13,69 +13,59 @@
namespace LIBC_NAMESPACE_DECL {
-/// A FIFO free-list storing Blocks of the same size.
+/// A circularly-linked FIFO list node storing a free Block. A list is a
+/// FreeList2*; nullptr is an empty list. All Blocks on a list are the same
+/// size.
class FreeList2 {
-protected:
- // A circular doubly-linked node.
- struct Node {
- Node *prev;
- Node *next;
- };
-
public:
- // Blocks with inner sizes smaller than this must not be pushed.
- static constexpr size_t MIN_INNER_SIZE = sizeof(Node);
-
- bool empty() const { return !begin_; }
- Block<> *front() const;
+ Block<> *block() const {
+ return const_cast<Block<> *>(Block<>::from_usable_space(this));
+ }
- /// Push to the back.
- void push(Block<> *block);
+ /// Push to the back. The Block must be able to contain a FreeList2.
+ static void push(FreeList2 *&list, Block<> *block);
/// Pop the front.
- void pop();
+ static void pop(FreeList2 *&list);
protected:
/// Push an already-constructed node to the back.
- void push(Node *node);
+ static void push(FreeList2 *&list, FreeList2 *node);
private:
- Node *begin_ = nullptr;
+ // Circularly linked pointers to adjacent nodes.
+ FreeList2 *prev;
+ FreeList2 *next;
};
-LIBC_INLINE Block<> *FreeList2::front() const {
- LIBC_ASSERT(!empty());
- return Block<>::from_usable_space(begin_);
-}
-
-LIBC_INLINE void FreeList2::push(Block<> *block) {
- LIBC_ASSERT(block->inner_size() >= MIN_INNER_SIZE &&
+LIBC_INLINE void FreeList2::push(FreeList2 *&list, Block<> *block) {
+ LIBC_ASSERT(block->inner_size() >= sizeof(FreeList2) &&
"block too small to accomodate free list node");
- push(new (block->usable_space()) Node);
+ push(list, new (block->usable_space()) FreeList2);
}
-LIBC_INLINE void FreeList2::pop() {
- LIBC_ASSERT(!empty());
- if (begin_->next == begin_) {
- begin_ = nullptr;
+LIBC_INLINE void FreeList2::pop(FreeList2 *&list) {
+ LIBC_ASSERT(list != nullptr && "cannot pop from empty list");
+ if (list->next == list) {
+ list = nullptr;
} else {
- begin_->prev->next = begin_->next;
- begin_->next->prev = begin_->prev;
- begin_ = begin_->next;
+ list->prev->next = list->next;
+ list->next->prev = list->prev;
+ list = list->next;
}
}
-LIBC_INLINE void FreeList2::push(Node *node) {
- if (begin_) {
+LIBC_INLINE void FreeList2::push(FreeList2 *&list, FreeList2 *node) {
+ if (list) {
LIBC_ASSERT(Block<>::from_usable_space(node)->outer_size() ==
- front()->outer_size() &&
+ list->block()->outer_size() &&
"freelist entries must have the same size");
- node->prev = begin_->prev;
- node->next = begin_;
- begin_->prev->next = node;
- begin_->prev = node;
+ node->prev = list->prev;
+ node->next = list;
+ list->prev->next = node;
+ list->prev = node;
} else {
- begin_ = node->prev = node->next = node;
+ list = node->prev = node->next = node;
}
}
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index c1878cfafb36cb..2fa3b90594a0a2 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -13,22 +13,10 @@
namespace LIBC_NAMESPACE_DECL {
-/// A trie representing a map of free lists covering a contiguous SizeRange.
+/// A trie node containing a free list. The subtrie contains a contiguous
+/// SizeRange of freelists.There is no relationship between the size of this
+/// free list and the size ranges of the subtries.
class FreeTrie : public FreeList2 {
-private:
- // A subtrie of free lists covering a contiguous SizeRange. This is also a
- // free list with a size somewhere within the range. There is no relationship
- // between the size of this free list and the sizes of the lower and upper
- // subtries.
- struct Node : public FreeList2::Node {
- // The containing trie or nullptr if this is the root.
- Node *parent;
- // The child subtrie covering the lower half of this subtrie's size range.
- Node *lower;
- // The child subtrie covering the upper half of this subtrie's size range.
- Node *upper;
- };
-
public:
// Power-of-two range of sizes covered by a subtrie.
class SizeRange {
@@ -49,15 +37,18 @@ class FreeTrie : public FreeList2 {
size_t width;
};
- // Blocks with inner sizes smaller than this must not be pushed.
- static constexpr size_t MIN_INNER_SIZE = sizeof(Node);
+ /// Push to the back of this node's free list.
+ static void push(FreeTrie *&trie, Block<> *block);
- /// Push to this freelist.
- void push(Block<> *block);
+ /// Pop from the front of this node's free list.
+ static void pop(FreeTrie *&trie);
- /// Push to the correctly-sized freelist. The caller must provide the size
- /// range for this trie, since it isn't stored within.
- void push(Block<> *block, SizeRange range);
+ // The containing trie or nullptr if this is the root.
+ FreeTrie *parent;
+ // The child subtrie covering the lower half of this subtrie's size range.
+ FreeTrie *lower;
+ // The child subtrie covering the upper half of this subtrie's size range.
+ FreeTrie *upper;
};
LIBC_INLINE FreeTrie::SizeRange::SizeRange(size_t min, size_t width)
@@ -75,22 +66,28 @@ LIBC_INLINE size_t FreeTrie::SizeRange::middle() const {
return min + width / 2;
}
-LIBC_INLINE void FreeTrie::push(Block<> *block) {
- LIBC_ASSERT(block->inner_size() >= MIN_INNER_SIZE &&
- "block too small to accomodate free list node");
- Node *node = new (block->usable_space()) Node;
- if (empty())
+LIBC_INLINE void FreeTrie::push(FreeTrie *&trie, Block<> *block) {
+ LIBC_ASSERT(block->inner_size() >= sizeof(FreeTrie) &&
+ "block too small to accomodate free trie node");
+ FreeTrie *node = new (block->usable_space()) FreeTrie;
+ // The trie links are irrelevant for all but the first node in the free list.
+ if (!trie)
node->parent = node->lower = node->upper = nullptr;
- FreeList2::push(node);
+ FreeList2 *list = trie;
+ FreeList2::push(list, node);
+ trie = static_cast<FreeTrie *>(list);
}
-LIBC_INLINE void FreeTrie::push(Block<> *block, SizeRange range) {
- if (empty() || block->outer_size() == front()->outer_size()) {
- push(block);
- return;
+LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
+ FreeList2 *list = trie;
+ FreeList2::pop(list);
+ FreeTrie *new_trie = static_cast<FreeTrie*>(list);
+ if (new_trie) {
+ new_trie->parent = trie->parent;
+ new_trie->lower = trie->lower;
+ new_trie->upper = trie->upper;
}
-
- // TODO;
+ trie = new_trie;
}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/freelist2_test.cpp b/libc/test/src/__support/freelist2_test.cpp
index abac9b11b8153f..2cedf67fcfec35 100644
--- a/libc/test/src/__support/freelist2_test.cpp
+++ b/libc/test/src/__support/freelist2_test.cpp
@@ -13,11 +13,6 @@
namespace LIBC_NAMESPACE_DECL {
-TEST(LlvmLibcFreeList2, Contruct) {
- FreeList2 list;
- EXPECT_TRUE(list.empty());
-}
-
TEST(LlvmLibcFreeList2, PushPop) {
cpp::byte mem1[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem1);
@@ -29,162 +24,17 @@ TEST(LlvmLibcFreeList2, PushPop) {
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block2 = *maybeBlock;
- FreeList2 list;
- list.push(block1);
- ASSERT_FALSE(list.empty());
- EXPECT_EQ(list.front(), block1);
- list.push(block2);
- EXPECT_EQ(list.front(), block1);
- list.pop();
- ASSERT_FALSE(list.empty());
- EXPECT_EQ(list.front(), block2);
- list.pop();
- EXPECT_TRUE(list.empty());
-}
-
-#if 0
-TEST(LlvmLibcFreeList2, EmptyListHasNoMembers) {
- FreeList<SIZE> list(example_sizes);
-
- auto item = list.find_chunk(4);
- EXPECT_EQ(item.size(), static_cast<size_t>(0));
- item = list.find_chunk(128);
- EXPECT_EQ(item.size(), static_cast<size_t>(0));
-}
-
-TEST(LlvmLibcFreeList, CanRetrieveAddedMember) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t N = 512;
-
- byte data[N] = {byte(0)};
-
- bool ok = list.add_chunk(span<byte>(data, N));
- EXPECT_TRUE(ok);
-
- auto item = list.find_chunk(N);
- EXPECT_EQ(item.size(), N);
- EXPECT_EQ(item.data(), data);
-}
-
-TEST(LlvmLibcFreeList, CanRetrieveAddedMemberForSmallerSize) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t N = 512;
-
- byte data[N] = {byte(0)};
-
- ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
- auto item = list.find_chunk(N / 2);
- EXPECT_EQ(item.size(), N);
- EXPECT_EQ(item.data(), data);
-}
-
-TEST(LlvmLibcFreeList, CanRemoveItem) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t N = 512;
-
- byte data[N] = {byte(0)};
-
- ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
- EXPECT_TRUE(list.remove_chunk(span<byte>(data, N)));
-
- auto item = list.find_chunk(N);
- EXPECT_EQ(item.size(), static_cast<size_t>(0));
-}
-
-TEST(LlvmLibcFreeList, FindReturnsSmallestChunk) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t kN1 = 512;
- constexpr size_t kN2 = 1024;
-
- byte data1[kN1] = {byte(0)};
- byte data2[kN2] = {byte(0)};
-
- ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
- ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));
-
- auto chunk = list.find_chunk(kN1 / 2);
- EXPECT_EQ(chunk.size(), kN1);
- EXPECT_EQ(chunk.data(), data1);
-
- chunk = list.find_chunk(kN1);
- EXPECT_EQ(chunk.size(), kN1);
- EXPECT_EQ(chunk.data(), data1);
-
- chunk = list.find_chunk(kN1 + 1);
- EXPECT_EQ(chunk.size(), kN2);
- EXPECT_EQ(chunk.data(), data2);
-}
-
-TEST(LlvmLibcFreeList, FindReturnsCorrectChunkInSameBucket) {
- // If we have two values in the same bucket, ensure that the allocation will
- // pick an appropriately sized one.
- FreeList<SIZE> list(example_sizes);
- constexpr size_t kN1 = 512;
- constexpr size_t kN2 = 257;
-
- byte data1[kN1] = {byte(0)};
- byte data2[kN2] = {byte(0)};
-
- // List should now be 257 -> 512 -> NULL
- ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
- ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));
-
- auto chunk = list.find_chunk(kN2 + 1);
- EXPECT_EQ(chunk.size(), kN1);
-}
-
-TEST(LlvmLibcFreeList, FindCanMoveUpThroughBuckets) {
- // Ensure that finding a chunk will move up through buckets if no appropriate
- // chunks were found in a given bucket
- FreeList<SIZE> list(example_sizes);
- constexpr size_t kN1 = 257;
- constexpr size_t kN2 = 513;
-
- byte data1[kN1] = {byte(0)};
- byte data2[kN2] = {byte(0)};
-
- // List should now be:
- // bkt[3] (257 bytes up to 512 bytes) -> 257 -> NULL
- // bkt[4] (513 bytes up to 1024 bytes) -> 513 -> NULL
- ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
- ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));
-
- // Request a 300 byte chunk. This should return the 513 byte one
- auto chunk = list.find_chunk(kN1 + 1);
- EXPECT_EQ(chunk.size(), kN2);
-}
-
-TEST(LlvmLibcFreeList, RemoveUnknownChunkReturnsNotFound) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t N = 512;
-
- byte data[N] = {byte(0)};
- byte data2[N] = {byte(0)};
-
- ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
- EXPECT_FALSE(list.remove_chunk(span<byte>(data2, N)));
-}
-
-TEST(LlvmLibcFreeList, CanStoreMultipleChunksPerBucket) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t N = 512;
-
- byte data1[N] = {byte(0)};
- byte data2[N] = {byte(0)};
-
- ASSERT_TRUE(list.add_chunk(span<byte>(data1, N)));
- ASSERT_TRUE(list.add_chunk(span<byte>(data2, N)));
-
- auto chunk1 = list.find_chunk(N);
- ASSERT_TRUE(list.remove_chunk(chunk1));
- auto chunk2 = list.find_chunk(N);
- ASSERT_TRUE(list.remove_chunk(chunk2));
-
- // Ordering of the chunks doesn't matter
- EXPECT_TRUE(chunk1.data() != chunk2.data());
- EXPECT_TRUE(chunk1.data() == data1 || chunk1.data() == data2);
- EXPECT_TRUE(chunk2.data() == data1 || chunk2.data() == data2);
+ FreeList2 *list = nullptr;
+ FreeList2::push(list, block1);
+ ASSERT_NE(list, static_cast<FreeList2*>(nullptr));
+ EXPECT_EQ(list->block(), block1);
+ FreeList2::push(list, block2);
+ EXPECT_EQ(list->block(), block1);
+ FreeList2::pop(list);
+ ASSERT_NE(list, static_cast<FreeList2*>(nullptr));
+ EXPECT_EQ(list->block(), block2);
+ FreeList2::pop(list);
+ ASSERT_EQ(list, static_cast<FreeList2*>(nullptr));
}
-#endif
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index fef5d16bb89ac6..486c16f601fee6 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -13,12 +13,7 @@
namespace LIBC_NAMESPACE_DECL {
-TEST(LlvmLibcFreeTrie, Construct) {
- FreeTrie trie;
- EXPECT_TRUE(trie.empty());
-}
-
-TEST(LlvmLibcFreeList2, PushPopDirect) {
+TEST(LlvmLibcFreeTrie, PushPop) {
cpp::byte mem1[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem1);
ASSERT_TRUE(maybeBlock.has_value());
@@ -29,17 +24,17 @@ TEST(LlvmLibcFreeList2, PushPopDirect) {
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block2 = *maybeBlock;
- FreeList2 list;
- list.push(block1);
- ASSERT_FALSE(list.empty());
- EXPECT_EQ(list.front(), block1);
- list.push(block2);
- EXPECT_EQ(list.front(), block1);
- list.pop();
- ASSERT_FALSE(list.empty());
- EXPECT_EQ(list.front(), block2);
- list.pop();
- EXPECT_TRUE(list.empty());
+ FreeTrie *list = nullptr;
+ FreeTrie::push(list, block1);
+ ASSERT_NE(list, static_cast<FreeTrie *>(nullptr));
+ EXPECT_EQ(list->block(), block1);
+ FreeTrie::push(list, block2);
+ EXPECT_EQ(list->block(), block1);
+ FreeTrie::pop(list);
+ ASSERT_NE(list, static_cast<FreeTrie *>(nullptr));
+ EXPECT_EQ(list->block(), block2);
+ FreeTrie::pop(list);
+ ASSERT_EQ(list, static_cast<FreeTrie *>(nullptr));
}
} // namespace LIBC_NAMESPACE_DECL
>From 34889617c72cb9be321d0fbe9b0ffb319829f72d Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 7 Aug 2024 17:01:55 -0700
Subject: [PATCH 12/54] Trivial find for sub tries
---
libc/src/__support/freetrie.h | 11 +++++-
libc/test/src/__support/freelist2_test.cpp | 3 ++
libc/test/src/__support/freetrie_test.cpp | 41 ++++++++++++++++------
3 files changed, 43 insertions(+), 12 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 2fa3b90594a0a2..95dca81bb8df8a 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -43,6 +43,11 @@ class FreeTrie : public FreeList2 {
/// Pop from the front of this node's free list.
static void pop(FreeTrie *&trie);
+ /// Finds the free trie for a given size. This may be a referance to a nullptr
+ /// at the correct place in the trie structure. The caller must provide the
+ /// SizeRange for this trie; the trie does not store it.
+ static FreeTrie *&find(FreeTrie *&trie, size_t size, SizeRange range);
+
// The containing trie or nullptr if this is the root.
FreeTrie *parent;
// The child subtrie covering the lower half of this subtrie's size range.
@@ -81,7 +86,7 @@ LIBC_INLINE void FreeTrie::push(FreeTrie *&trie, Block<> *block) {
LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
FreeList2 *list = trie;
FreeList2::pop(list);
- FreeTrie *new_trie = static_cast<FreeTrie*>(list);
+ FreeTrie *new_trie = static_cast<FreeTrie *>(list);
if (new_trie) {
new_trie->parent = trie->parent;
new_trie->lower = trie->lower;
@@ -90,6 +95,10 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
trie = new_trie;
}
+FreeTrie *&FreeTrie::find(FreeTrie *&list, size_t size, SizeRange range) {
+ return list;
+}
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
diff --git a/libc/test/src/__support/freelist2_test.cpp b/libc/test/src/__support/freelist2_test.cpp
index 2cedf67fcfec35..a6418883d65503 100644
--- a/libc/test/src/__support/freelist2_test.cpp
+++ b/libc/test/src/__support/freelist2_test.cpp
@@ -28,11 +28,14 @@ TEST(LlvmLibcFreeList2, PushPop) {
FreeList2::push(list, block1);
ASSERT_NE(list, static_cast<FreeList2*>(nullptr));
EXPECT_EQ(list->block(), block1);
+
FreeList2::push(list, block2);
EXPECT_EQ(list->block(), block1);
+
FreeList2::pop(list);
ASSERT_NE(list, static_cast<FreeList2*>(nullptr));
EXPECT_EQ(list->block(), block2);
+
FreeList2::pop(list);
ASSERT_EQ(list, static_cast<FreeList2*>(nullptr));
}
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 486c16f601fee6..9b70ef4c4d9bf6 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -24,17 +24,36 @@ TEST(LlvmLibcFreeTrie, PushPop) {
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block2 = *maybeBlock;
- FreeTrie *list = nullptr;
- FreeTrie::push(list, block1);
- ASSERT_NE(list, static_cast<FreeTrie *>(nullptr));
- EXPECT_EQ(list->block(), block1);
- FreeTrie::push(list, block2);
- EXPECT_EQ(list->block(), block1);
- FreeTrie::pop(list);
- ASSERT_NE(list, static_cast<FreeTrie *>(nullptr));
- EXPECT_EQ(list->block(), block2);
- FreeTrie::pop(list);
- ASSERT_EQ(list, static_cast<FreeTrie *>(nullptr));
+ FreeTrie *trie = nullptr;
+ FreeTrie::push(trie, block1);
+ ASSERT_NE(trie, static_cast<FreeTrie *>(nullptr));
+ EXPECT_EQ(trie->block(), block1);
+
+ FreeTrie::push(trie, block2);
+ EXPECT_EQ(trie->block(), block1);
+
+ FreeTrie::pop(trie);
+ ASSERT_NE(trie, static_cast<FreeTrie *>(nullptr));
+ EXPECT_EQ(trie->block(), block2);
+
+ FreeTrie::pop(trie);
+ ASSERT_EQ(trie, static_cast<FreeTrie *>(nullptr));
+}
+
+TEST(LlvmLibcFreeTrie, Find) {
+ FreeTrie *trie = nullptr;
+ FreeTrie *&empty_found = FreeTrie::find(trie, 123, {0, 1024});
+ EXPECT_EQ(&empty_found, &trie);
+
+ cpp::byte mem1[1024];
+ optional<Block<> *> maybeBlock = Block<>::init(mem1);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *block1 = *maybeBlock;
+
+ FreeTrie::push(trie, block1);
+
+ FreeTrie *&root_found = FreeTrie::find(trie, block1->inner_size(), {0, 1024});
+ EXPECT_EQ(&root_found, &trie);
}
} // namespace LIBC_NAMESPACE_DECL
>From 6bd64b4341328570d0c12dd96f511028a91f72c6 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 7 Aug 2024 17:18:20 -0700
Subject: [PATCH 13/54] Add simple (but complete!) trie find
---
libc/src/__support/freelist2.h | 3 +++
libc/src/__support/freetrie.h | 22 ++++++++++++++++++++--
libc/test/src/__support/freetrie_test.cpp | 14 ++++++++++++--
3 files changed, 35 insertions(+), 4 deletions(-)
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index 1bedc9de8424c7..3a1a470f7872ca 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -22,6 +22,9 @@ class FreeList2 {
return const_cast<Block<> *>(Block<>::from_usable_space(this));
}
+ /// @returns Size for all blocks on the list.
+ size_t size() const { return block()->inner_size(); }
+
/// Push to the back. The Block must be able to contain a FreeList2.
static void push(FreeList2 *&list, Block<> *block);
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 95dca81bb8df8a..d605be802d18f5 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -32,6 +32,8 @@ class FreeTrie : public FreeList2 {
/// @returns The split point between lower and upper.
size_t middle() const;
+ bool contains(size_t size) const;
+
private:
size_t min;
size_t width;
@@ -64,13 +66,23 @@ LIBC_INLINE FreeTrie::SizeRange::SizeRange(size_t min, size_t width)
LIBC_INLINE FreeTrie::SizeRange FreeTrie::SizeRange::lower() const {
return {min, width / 2};
}
+
LIBC_INLINE FreeTrie::SizeRange FreeTrie::SizeRange::upper() const {
return {middle(), width / 2};
}
+
LIBC_INLINE size_t FreeTrie::SizeRange::middle() const {
return min + width / 2;
}
+LIBC_INLINE bool FreeTrie::SizeRange::contains(size_t size) const {
+ if (size < min)
+ return false;
+ if (size > min + width)
+ return false;
+ return true;
+}
+
LIBC_INLINE void FreeTrie::push(FreeTrie *&trie, Block<> *block) {
LIBC_ASSERT(block->inner_size() >= sizeof(FreeTrie) &&
"block too small to accomodate free trie node");
@@ -91,12 +103,18 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
new_trie->parent = trie->parent;
new_trie->lower = trie->lower;
new_trie->upper = trie->upper;
+ } else {
+ // TODO
}
trie = new_trie;
}
-FreeTrie *&FreeTrie::find(FreeTrie *&list, size_t size, SizeRange range) {
- return list;
+FreeTrie *&FreeTrie::find(FreeTrie *&trie, size_t size, SizeRange range) {
+ LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
+ if (!trie || trie->size() == size)
+ return trie;
+ return find(size <= range.middle() ? trie->lower : trie->upper, size,
+ size <= range.middle() ? range.lower() : range.upper());
}
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 9b70ef4c4d9bf6..24c45beceffd56 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -41,8 +41,10 @@ TEST(LlvmLibcFreeTrie, PushPop) {
}
TEST(LlvmLibcFreeTrie, Find) {
+ size_t WIDTH = 1024;
+
FreeTrie *trie = nullptr;
- FreeTrie *&empty_found = FreeTrie::find(trie, 123, {0, 1024});
+ FreeTrie *&empty_found = FreeTrie::find(trie, 123, {0, WIDTH});
EXPECT_EQ(&empty_found, &trie);
cpp::byte mem1[1024];
@@ -52,8 +54,16 @@ TEST(LlvmLibcFreeTrie, Find) {
FreeTrie::push(trie, block1);
- FreeTrie *&root_found = FreeTrie::find(trie, block1->inner_size(), {0, 1024});
+ FreeTrie *&root_found =
+ FreeTrie::find(trie, block1->inner_size(), {0, WIDTH});
EXPECT_EQ(&root_found, &trie);
+
+ FreeTrie *&less_found = FreeTrie::find(trie, WIDTH / 2, {0, 1024});
+ EXPECT_NE(&less_found, &trie);
+
+ FreeTrie *&greater_found = FreeTrie::find(trie, WIDTH / 2 + 1, {0, 1024});
+ EXPECT_NE(&greater_found, &trie);
+ EXPECT_NE(&greater_found, &less_found);
}
} // namespace LIBC_NAMESPACE_DECL
>From 80571fe7f04f660224c6f484db4c9ac97103c893 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Thu, 8 Aug 2024 12:09:20 -0700
Subject: [PATCH 14/54] Assert that found links are nullptr for a root push
---
libc/test/src/__support/freetrie_test.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 24c45beceffd56..c7ffe29e017926 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -60,10 +60,12 @@ TEST(LlvmLibcFreeTrie, Find) {
FreeTrie *&less_found = FreeTrie::find(trie, WIDTH / 2, {0, 1024});
EXPECT_NE(&less_found, &trie);
+ EXPECT_EQ(less_found, static_cast<FreeTrie *>(nullptr));
FreeTrie *&greater_found = FreeTrie::find(trie, WIDTH / 2 + 1, {0, 1024});
EXPECT_NE(&greater_found, &trie);
EXPECT_NE(&greater_found, &less_found);
+ EXPECT_EQ(greater_found, static_cast<FreeTrie *>(nullptr));
}
} // namespace LIBC_NAMESPACE_DECL
>From eb90cc2bbea947a0c0184fdd9a6c6fd48b26c97b Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Thu, 8 Aug 2024 12:17:29 -0700
Subject: [PATCH 15/54] Add a test that pop preserves children
---
libc/test/src/__support/freetrie_test.cpp | 34 +++++++++++++++++++++--
1 file changed, 32 insertions(+), 2 deletions(-)
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index c7ffe29e017926..27fd7da821b8fc 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -47,8 +47,8 @@ TEST(LlvmLibcFreeTrie, Find) {
FreeTrie *&empty_found = FreeTrie::find(trie, 123, {0, WIDTH});
EXPECT_EQ(&empty_found, &trie);
- cpp::byte mem1[1024];
- optional<Block<> *> maybeBlock = Block<>::init(mem1);
+ cpp::byte mem[1024];
+ optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block1 = *maybeBlock;
@@ -68,4 +68,34 @@ TEST(LlvmLibcFreeTrie, Find) {
EXPECT_EQ(greater_found, static_cast<FreeTrie *>(nullptr));
}
+TEST(LlvmLibcFreeTrie, RootPopPreservesChild) {
+ cpp::byte mem1[1024];
+ optional<Block<> *> maybeBlock = Block<>::init(mem1);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *block1 = *maybeBlock;
+
+ cpp::byte mem2[1024];
+ maybeBlock = Block<>::init(mem2);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *block2 = *maybeBlock;
+
+ cpp::byte mem3[2048];
+ maybeBlock = Block<>::init(mem3);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *block3 = *maybeBlock;
+
+ FreeTrie *trie = nullptr;
+ FreeTrie::push(trie, block1);
+ FreeTrie::push(trie, block2);
+
+ FreeTrie *&child = trie->find(trie, block3->inner_size(), {0, 2048});
+ FreeTrie::push(child, block3);
+ ASSERT_NE(child, static_cast<FreeTrie *>(nullptr));
+ EXPECT_EQ(child->block(), block3);
+
+ FreeTrie::pop(trie);
+ FreeTrie *&new_child = trie->find(trie, block3->inner_size(), {0, 2048});
+ EXPECT_EQ(new_child, child);
+}
+
} // namespace LIBC_NAMESPACE_DECL
>From fff9e600a8e7d7861edbca3a43280be4592f2f85 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Thu, 8 Aug 2024 13:50:33 -0700
Subject: [PATCH 16/54] More sophisticated testing for pop
---
libc/src/__support/freetrie.h | 37 +++++++++++++----
libc/test/src/__support/freetrie_test.cpp | 48 +++++++++++++++++++----
2 files changed, 71 insertions(+), 14 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index d605be802d18f5..88a40b1c4fcca6 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -50,8 +50,10 @@ class FreeTrie : public FreeList2 {
/// SizeRange for this trie; the trie does not store it.
static FreeTrie *&find(FreeTrie *&trie, size_t size, SizeRange range);
- // The containing trie or nullptr if this is the root.
- FreeTrie *parent;
+private:
+ /// Return an abitrary leaf.
+ FreeTrie &leaf();
+
// The child subtrie covering the lower half of this subtrie's size range.
FreeTrie *lower;
// The child subtrie covering the upper half of this subtrie's size range.
@@ -89,7 +91,7 @@ LIBC_INLINE void FreeTrie::push(FreeTrie *&trie, Block<> *block) {
FreeTrie *node = new (block->usable_space()) FreeTrie;
// The trie links are irrelevant for all but the first node in the free list.
if (!trie)
- node->parent = node->lower = node->upper = nullptr;
+ node->lower = node->upper = nullptr;
FreeList2 *list = trie;
FreeList2::push(list, node);
trie = static_cast<FreeTrie *>(list);
@@ -100,13 +102,27 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
FreeList2::pop(list);
FreeTrie *new_trie = static_cast<FreeTrie *>(list);
if (new_trie) {
- new_trie->parent = trie->parent;
+ // The freelist is non-empty, so copy the trie links to the new head.
new_trie->lower = trie->lower;
new_trie->upper = trie->upper;
- } else {
- // TODO
+ trie = new_trie;
+ return;
+ }
+
+ // The freelist is empty.
+
+ FreeTrie &l = trie->leaf();
+ if (&l == trie) {
+ // The last element of the trie was remved.
+ trie = nullptr;
+ return;
}
- trie = new_trie;
+
+ // Replace the root with an arbitrary leaf. This is legal because there is
+ // no relationship between the size of the root and its children.
+ l.lower = trie->lower;
+ l.upper = trie->upper;
+ trie = &l;
}
FreeTrie *&FreeTrie::find(FreeTrie *&trie, size_t size, SizeRange range) {
@@ -117,6 +133,13 @@ FreeTrie *&FreeTrie::find(FreeTrie *&trie, size_t size, SizeRange range) {
size <= range.middle() ? range.lower() : range.upper());
}
+FreeTrie &FreeTrie::leaf() {
+ FreeTrie *t = this;
+ while (t->lower || t->upper)
+ t = t->lower ? t->lower : t->upper;
+ return *t;
+}
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 27fd7da821b8fc..2d611e7e90f026 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -68,7 +68,7 @@ TEST(LlvmLibcFreeTrie, Find) {
EXPECT_EQ(greater_found, static_cast<FreeTrie *>(nullptr));
}
-TEST(LlvmLibcFreeTrie, RootPopPreservesChild) {
+TEST(LlvmLibcFreeTrie, RootPopWithChild) {
cpp::byte mem1[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem1);
ASSERT_TRUE(maybeBlock.has_value());
@@ -84,18 +84,52 @@ TEST(LlvmLibcFreeTrie, RootPopPreservesChild) {
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block3 = *maybeBlock;
+ cpp::byte mem4[2047];
+ maybeBlock = Block<>::init(mem4);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *block4 = *maybeBlock;
+
FreeTrie *trie = nullptr;
FreeTrie::push(trie, block1);
FreeTrie::push(trie, block2);
- FreeTrie *&child = trie->find(trie, block3->inner_size(), {0, 2048});
- FreeTrie::push(child, block3);
- ASSERT_NE(child, static_cast<FreeTrie *>(nullptr));
- EXPECT_EQ(child->block(), block3);
+ FreeTrie *&child3 = trie->find(trie, block3->inner_size(), {0, 4096});
+ FreeTrie::push(child3, block3);
+
+ ASSERT_NE(child3, static_cast<FreeTrie *>(nullptr));
+ EXPECT_EQ(child3->block(), block3);
+
+ FreeTrie *&child4 = trie->find(trie, block4->inner_size(), {0, 4096});
+ FreeTrie::push(child4, block4);
+ ASSERT_NE(child4, static_cast<FreeTrie *>(nullptr));
+ EXPECT_EQ(child4->block(), block4);
+
+ // Expected Trie:
+ // block1 -> block2
+ // lower:
+ // block3
+ // upper:
+ // block4
+
+ FreeTrie::pop(trie);
+ FreeTrie *&new_child4 = trie->find(trie, block4->inner_size(), {0, 4096});
+ // Expected Trie:
+ // block2
+ // lower:
+ // block3
+ // upper:
+ // block4
+ EXPECT_EQ(new_child4, child4);
FreeTrie::pop(trie);
- FreeTrie *&new_child = trie->find(trie, block3->inner_size(), {0, 2048});
- EXPECT_EQ(new_child, child);
+
+ // Expected Trie:
+ // block4
+ // lower:
+ // block3
+ EXPECT_EQ(trie, child4);
+ FreeTrie *&new_child3 = trie->find(trie, block3->inner_size(), {0, 4096});
+ EXPECT_EQ(new_child3, child3);
}
} // namespace LIBC_NAMESPACE_DECL
>From 2c39b49712ded9379a12b3ae126ba198c2a473d0 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 13 Aug 2024 12:55:06 -0700
Subject: [PATCH 17/54] Find best fit function (untested)
---
libc/src/__support/freetrie.h | 87 +++++++++++++++++++++++++++++------
1 file changed, 72 insertions(+), 15 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 88a40b1c4fcca6..d1b381bae7d28e 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -19,8 +19,10 @@ namespace LIBC_NAMESPACE_DECL {
class FreeTrie : public FreeList2 {
public:
// Power-of-two range of sizes covered by a subtrie.
- class SizeRange {
- public:
+ struct SizeRange {
+ size_t min;
+ size_t width;
+
SizeRange(size_t min, size_t width);
/// @returns The lower half of the size range.
@@ -33,10 +35,6 @@ class FreeTrie : public FreeList2 {
size_t middle() const;
bool contains(size_t size) const;
-
- private:
- size_t min;
- size_t width;
};
/// Push to the back of this node's free list.
@@ -50,7 +48,13 @@ class FreeTrie : public FreeList2 {
/// SizeRange for this trie; the trie does not store it.
static FreeTrie *&find(FreeTrie *&trie, size_t size, SizeRange range);
+ static FreeTrie **find_best_fit(FreeTrie *&trie, size_t size,
+ SizeRange range);
+
private:
+ /// Return the smallest-sized free list in the trie.
+ static FreeTrie *&smallest(FreeTrie *&trie);
+
/// Return an abitrary leaf.
FreeTrie &leaf();
@@ -102,7 +106,7 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
FreeList2::pop(list);
FreeTrie *new_trie = static_cast<FreeTrie *>(list);
if (new_trie) {
- // The freelist is non-empty, so copy the trie links to the new head.
+ // The freelist is non-empty, so copy the trie links to the new head.
new_trie->lower = trie->lower;
new_trie->upper = trie->upper;
trie = new_trie;
@@ -127,17 +131,70 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
FreeTrie *&FreeTrie::find(FreeTrie *&trie, size_t size, SizeRange range) {
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
- if (!trie || trie->size() == size)
- return trie;
- return find(size <= range.middle() ? trie->lower : trie->upper, size,
- size <= range.middle() ? range.lower() : range.upper());
+ FreeTrie **cur = ≜
+ while (*cur && (*cur)->size() != size) {
+ LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
+ if (size <= range.middle()) {
+ cur = &(*cur)->lower;
+ range = range.lower();
+ } else {
+ cur = &(*cur)->upper;
+ range = range.upper();
+ }
+ }
+ return *cur;
+}
+
+FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
+ SizeRange range) {
+ LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
+ FreeTrie **cur = ≜
+ FreeTrie **skipped_upper_trie = nullptr;
+
+ // Inductively assume the best fit is in this subtrie.
+ while (*cur) {
+ LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
+ size_t cur_size = (*cur)->size();
+ if (cur_size == size)
+ return cur;
+ if (size <= range.middle()) {
+ // The lower subtrie has the requested size in its range. So, if it has at
+ // least one larger entry, the best fit is in the lower subtrie. But if
+ // the lower subtrie contains only smaller sizes, the best fit is in the
+ // larger trie. So keep track of it.
+ if ((*cur)->upper)
+ skipped_upper_trie = &(*cur)->upper;
+ cur = &(*cur)->lower;
+ range = range.lower();
+ } else {
+ // The lower child is too small, so the best fit is in the upper subtrie.
+ cur = &(*cur)->upper;
+ range = range.upper();
+ }
+ }
+
+ // A lower subtrie contained size in its range, but it had only entries
+ // smaller than size. Accordingly, the best fit is the smallest entry in the
+ // corresponding upper subtrie.
+ return &FreeTrie::smallest(*skipped_upper_trie);
}
FreeTrie &FreeTrie::leaf() {
- FreeTrie *t = this;
- while (t->lower || t->upper)
- t = t->lower ? t->lower : t->upper;
- return *t;
+ FreeTrie *cur = this;
+ while (cur->lower || cur->upper)
+ cur = cur->lower ? cur->lower : cur->upper;
+ return *cur;
+}
+
+FreeTrie *&FreeTrie::smallest(FreeTrie *&trie) {
+ FreeTrie **cur = ≜
+ FreeTrie **ret = nullptr;
+ while (*cur) {
+ if (!ret || (*cur)->size() < (*ret)->size())
+ ret = cur;
+ cur = (*cur)->lower ? &(*cur)->lower : &(*cur)->upper;
+ }
+ return *ret;
}
} // namespace LIBC_NAMESPACE_DECL
>From 500cad3d66d1a783caba202804a0b9d7bf3fd01a Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 14 Aug 2024 12:07:02 -0700
Subject: [PATCH 18/54] Range test
---
libc/src/__support/freetrie.h | 22 ++++++++--------------
libc/test/src/__support/freetrie_test.cpp | 19 +++++++++++++++++++
2 files changed, 27 insertions(+), 14 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index d1b381bae7d28e..ab3335a92f05f9 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -31,9 +31,8 @@ class FreeTrie : public FreeList2 {
/// @returns The lower half of the size range.
SizeRange upper() const;
- /// @returns The split point between lower and upper.
- size_t middle() const;
-
+ /// @returns Whether the range contains the given size.
+ /// Lower bound is inclusive, upper bound is exclusive.
bool contains(size_t size) const;
};
@@ -74,11 +73,7 @@ LIBC_INLINE FreeTrie::SizeRange FreeTrie::SizeRange::lower() const {
}
LIBC_INLINE FreeTrie::SizeRange FreeTrie::SizeRange::upper() const {
- return {middle(), width / 2};
-}
-
-LIBC_INLINE size_t FreeTrie::SizeRange::middle() const {
- return min + width / 2;
+ return {min + width / 2, width / 2};
}
LIBC_INLINE bool FreeTrie::SizeRange::contains(size_t size) const {
@@ -134,7 +129,7 @@ FreeTrie *&FreeTrie::find(FreeTrie *&trie, size_t size, SizeRange range) {
FreeTrie **cur = ≜
while (*cur && (*cur)->size() != size) {
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
- if (size <= range.middle()) {
+ if (range.lower().contains(size)) {
cur = &(*cur)->lower;
range = range.lower();
} else {
@@ -157,11 +152,10 @@ FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
size_t cur_size = (*cur)->size();
if (cur_size == size)
return cur;
- if (size <= range.middle()) {
- // The lower subtrie has the requested size in its range. So, if it has at
- // least one larger entry, the best fit is in the lower subtrie. But if
- // the lower subtrie contains only smaller sizes, the best fit is in the
- // larger trie. So keep track of it.
+ if (range.lower().contains(size)) {
+ // If the lower subtree has at least one entry >= size, the best fit is in
+ // the lower subtrie. But if the lower subtrie contains only smaller
+ // sizes, the best fit is in the larger trie. So keep track of it.
if ((*cur)->upper)
skipped_upper_trie = &(*cur)->upper;
cur = &(*cur)->lower;
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 2d611e7e90f026..16ce12be2181ee 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -132,4 +132,23 @@ TEST(LlvmLibcFreeTrie, RootPopWithChild) {
EXPECT_EQ(new_child3, child3);
}
+TEST(LlvmLibcFreeTrie, SizeRange) {
+ FreeTrie::SizeRange range(123, 1024);
+ EXPECT_EQ(range.min, size_t{123});
+ EXPECT_EQ(range.width, size_t{1024});
+
+ EXPECT_TRUE(range.contains(123));
+ EXPECT_TRUE(range.contains(123 + 1024 - 1));
+ EXPECT_FALSE(range.contains(123 - 1));
+ EXPECT_FALSE(range.contains(123 + 1024 + 1));
+
+ FreeTrie::SizeRange lower = range.lower();
+ EXPECT_EQ(lower.min, size_t{123});
+ EXPECT_EQ(lower.width, size_t{1024 / 2});
+
+ FreeTrie::SizeRange upper = range.upper();
+ EXPECT_EQ(upper.min, size_t{123 + 1024 / 2});
+ EXPECT_EQ(upper.width, size_t{1024 / 2});
+}
+
} // namespace LIBC_NAMESPACE_DECL
>From b7140325c7c4e0d39296f792805716dc7f1ba148 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Thu, 15 Aug 2024 10:41:59 -0700
Subject: [PATCH 19/54] Add inner_size_free to block
---
libc/src/__support/block.h | 20 ++++++++++++++++++--
1 file changed, 18 insertions(+), 2 deletions(-)
diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h
index 96021b99587c87..e1f6726b27f08e 100644
--- a/libc/src/__support/block.h
+++ b/libc/src/__support/block.h
@@ -174,16 +174,32 @@ class Block {
return inner_size - sizeof(prev_) + BLOCK_OVERHEAD;
}
- /// @returns The number of usable bytes inside the block.
+ /// @returns The number of usable bytes inside the block were it to be
+ /// allocated.
size_t inner_size() const {
if (!next())
return 0;
return inner_size(outer_size());
}
+ /// @returns The number of usable bytes inside a block with the given outer
+ /// size were it to be allocated.
static size_t inner_size(size_t outer_size) {
// The usable region includes the prev_ field of the next block.
- return outer_size - BLOCK_OVERHEAD + sizeof(prev_);
+ return inner_size_free(outer_size) + sizeof(prev_);
+ }
+
+ /// @returns The number of usable bytes inside the block if it remains free.
+ size_t inner_size_free() const {
+ if (!next())
+ return 0;
+ return inner_size_free(outer_size());
+ }
+
+ /// @returns The number of usable bytes inside a block with the given outer
+ /// size if it remains free.
+ static size_t inner_size_free(size_t outer_size) {
+ return outer_size - BLOCK_OVERHEAD;
}
/// @returns A pointer to the usable space inside this block.
>From 802fc5bcc32b31871275867d64986f61eaa05979 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Thu, 15 Aug 2024 10:43:29 -0700
Subject: [PATCH 20/54] Check for free inner size, not avail inner size
---
libc/src/__support/freelist2.h | 2 +-
libc/src/__support/freetrie.h | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index 3a1a470f7872ca..85e0b569ce5cb4 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -42,7 +42,7 @@ class FreeList2 {
};
LIBC_INLINE void FreeList2::push(FreeList2 *&list, Block<> *block) {
- LIBC_ASSERT(block->inner_size() >= sizeof(FreeList2) &&
+ LIBC_ASSERT(block->inner_size_free() >= sizeof(FreeList2) &&
"block too small to accomodate free list node");
push(list, new (block->usable_space()) FreeList2);
}
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index ab3335a92f05f9..4de7395504c075 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -85,7 +85,7 @@ LIBC_INLINE bool FreeTrie::SizeRange::contains(size_t size) const {
}
LIBC_INLINE void FreeTrie::push(FreeTrie *&trie, Block<> *block) {
- LIBC_ASSERT(block->inner_size() >= sizeof(FreeTrie) &&
+ LIBC_ASSERT(block->inner_size_free() >= sizeof(FreeTrie) &&
"block too small to accomodate free trie node");
FreeTrie *node = new (block->usable_space()) FreeTrie;
// The trie links are irrelevant for all but the first node in the free list.
>From 81e7e84fad5e1db236dacbc2c4885b257f23c4e0 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Thu, 15 Aug 2024 12:43:30 -0700
Subject: [PATCH 21/54] Refactor free trie test
---
libc/src/__support/freelist2.h | 5 +
libc/src/__support/freetrie.h | 81 ++++++++-----
libc/test/src/__support/freetrie_test.cpp | 134 ++++++++++------------
3 files changed, 119 insertions(+), 101 deletions(-)
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index 85e0b569ce5cb4..025a5df5dae0c9 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -16,6 +16,11 @@ namespace LIBC_NAMESPACE_DECL {
/// A circularly-linked FIFO list node storing a free Block. A list is a
/// FreeList2*; nullptr is an empty list. All Blocks on a list are the same
/// size.
+///
+/// Accessing free blocks in FIFO order maximizes the amount of time before a
+/// free block is reused. This in turn maximizes the number of opportunities for
+/// it to be coalesced with an adjacent block, which tends to reduce heap
+/// fragmentation.
class FreeList2 {
public:
Block<> *block() const {
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 4de7395504c075..1608a03599e056 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -31,6 +31,9 @@ class FreeTrie : public FreeList2 {
/// @returns The lower half of the size range.
SizeRange upper() const;
+ /// @returns The largest size in this range.
+ size_t max() const;
+
/// @returns Whether the range contains the given size.
/// Lower bound is inclusive, upper bound is exclusive.
bool contains(size_t size) const;
@@ -51,9 +54,6 @@ class FreeTrie : public FreeList2 {
SizeRange range);
private:
- /// Return the smallest-sized free list in the trie.
- static FreeTrie *&smallest(FreeTrie *&trie);
-
/// Return an abitrary leaf.
FreeTrie &leaf();
@@ -76,6 +76,8 @@ LIBC_INLINE FreeTrie::SizeRange FreeTrie::SizeRange::upper() const {
return {min + width / 2, width / 2};
}
+size_t FreeTrie::SizeRange::max() const { return min + (width - 1); }
+
LIBC_INLINE bool FreeTrie::SizeRange::contains(size_t size) const {
if (size < min)
return false;
@@ -142,35 +144,65 @@ FreeTrie *&FreeTrie::find(FreeTrie *&trie, size_t size, SizeRange range) {
FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
SizeRange range) {
+ if (!cur)
+ return trie;
+
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
FreeTrie **cur = ≜
- FreeTrie **skipped_upper_trie = nullptr;
+ FreeTrie **best_fit = nullptr;
+ FreeTrie **deferred_upper_trie = nullptr;
- // Inductively assume the best fit is in this subtrie.
- while (*cur) {
+ // Inductively assume all better fits than the current best are in the
+ // current subtrie.
+ while (true) {
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
+
+ // Consider whether the current node is a better fit.
size_t cur_size = (*cur)->size();
if (cur_size == size)
return cur;
- if (range.lower().contains(size)) {
- // If the lower subtree has at least one entry >= size, the best fit is in
- // the lower subtrie. But if the lower subtrie contains only smaller
- // sizes, the best fit is in the larger trie. So keep track of it.
- if ((*cur)->upper)
- skipped_upper_trie = &(*cur)->upper;
+ if (!best_fit || cur_size < (*best_fit)->size())
+ best_fit = cur;
+
+ // Determine which subtries might contain better fits.
+ bool lower_impossible = !(*cur)->lower || range.lower().max() < size;
+ bool upper_impossible =
+ !(*cur)->upper ||
+ (best_fit && range.upper().min >= (*best_fit)->size());
+
+ if (lower_impossible && upper_impossible)
+ break;
+ if (lower_impossible) {
+ cur = &(*cur)->upper;
+ range = range.upper();
+ } else if (upper_impossible) {
cur = &(*cur)->lower;
range = range.lower();
} else {
- // The lower child is too small, so the best fit is in the upper subtrie.
- cur = &(*cur)->upper;
- range = range.upper();
+ // Both subtries might contain a better fit. But, any fit in the lower
+ // subtrie is better than the best fit in the upper subtrie. Accordingly,
+ // we can scan the lower subtrie, then return to the upper one if
+ // necessary.
+ cur = &(*cur)->lower;
+ range = range.lower();
+
+ // If we have already deferred a subtrie, it was an upper
+ // subtrie of its parent, and the node being deferred here is somewhere in
+ // the lower subtrie. That means that the new subtrie is strictly better
+ // than the old, and the old can be summarily ignored.
+ deferred_upper_trie = &(*cur)->upper;
}
}
- // A lower subtrie contained size in its range, but it had only entries
- // smaller than size. Accordingly, the best fit is the smallest entry in the
- // corresponding upper subtrie.
- return &FreeTrie::smallest(*skipped_upper_trie);
+ if (!deferred_upper_trie)
+ return best_fit;
+
+ // We have deferred an upper subtrie, and this is the only subtrie left that
+ // could contain a better fit, via the above analysis. It's also the case that
+ // any node within this subtrie is a fit, so just find the smallest.
+ cur = deferred_upper_trie;
+ while (*cur) {
+ }
}
FreeTrie &FreeTrie::leaf() {
@@ -180,17 +212,6 @@ FreeTrie &FreeTrie::leaf() {
return *cur;
}
-FreeTrie *&FreeTrie::smallest(FreeTrie *&trie) {
- FreeTrie **cur = ≜
- FreeTrie **ret = nullptr;
- while (*cur) {
- if (!ret || (*cur)->size() < (*ret)->size())
- ret = cur;
- cur = (*cur)->lower ? &(*cur)->lower : &(*cur)->upper;
- }
- return *ret;
-}
-
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 16ce12be2181ee..b1487a88b99b38 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -13,120 +13,112 @@
namespace LIBC_NAMESPACE_DECL {
-TEST(LlvmLibcFreeTrie, PushPop) {
- cpp::byte mem1[1024];
- optional<Block<> *> maybeBlock = Block<>::init(mem1);
- ASSERT_TRUE(maybeBlock.has_value());
- Block<> *block1 = *maybeBlock;
+template <size_t size> struct BlockMem {
+ BlockMem() {
+ optional<Block<> *> maybeBlock = Block<>::init(mem);
+ LIBC_ASSERT(maybeBlock.has_value() && "could not create test block");
+ block = *maybeBlock;
+ }
+ __attribute__((aligned(alignof(Block<>)))) cpp::byte mem[size];
+ Block<> *block;
+};
- cpp::byte mem2[1024];
- maybeBlock = Block<>::init(mem2);
- ASSERT_TRUE(maybeBlock.has_value());
- Block<> *block2 = *maybeBlock;
+TEST(LlvmLibcFreeTrie, PushPop) {
+ BlockMem<1024> block1_mem;
+ Block<> *block1 = block1_mem.block;
+ BlockMem<1024> block2_mem;
+ Block<> *block2 = block2_mem.block;
+ BlockMem<1024> block3_mem;
+ Block<> *block3 = block3_mem.block;
FreeTrie *trie = nullptr;
FreeTrie::push(trie, block1);
ASSERT_NE(trie, static_cast<FreeTrie *>(nullptr));
EXPECT_EQ(trie->block(), block1);
+ // Pushing blocks doesn't change the next block.
FreeTrie::push(trie, block2);
EXPECT_EQ(trie->block(), block1);
+ FreeTrie::push(trie, block3);
+ EXPECT_EQ(trie->block(), block1);
+ // Blocks are popped in FIFO order
FreeTrie::pop(trie);
ASSERT_NE(trie, static_cast<FreeTrie *>(nullptr));
EXPECT_EQ(trie->block(), block2);
+ FreeTrie::pop(trie);
+ ASSERT_NE(trie, static_cast<FreeTrie *>(nullptr));
+ EXPECT_EQ(trie->block(), block3);
+ // Popping the last block clears the list.
FreeTrie::pop(trie);
ASSERT_EQ(trie, static_cast<FreeTrie *>(nullptr));
}
TEST(LlvmLibcFreeTrie, Find) {
- size_t WIDTH = 1024;
-
+ // Finding in an empty trie returns the trie itself.
FreeTrie *trie = nullptr;
- FreeTrie *&empty_found = FreeTrie::find(trie, 123, {0, WIDTH});
+ FreeTrie *&empty_found = FreeTrie::find(trie, 123, {0, 1024});
EXPECT_EQ(&empty_found, &trie);
- cpp::byte mem[1024];
- optional<Block<> *> maybeBlock = Block<>::init(mem);
- ASSERT_TRUE(maybeBlock.has_value());
- Block<> *block1 = *maybeBlock;
-
- FreeTrie::push(trie, block1);
+ BlockMem<768> block_mem;
+ Block<> *block = block_mem.block;
+ FreeTrie::push(trie, block);
- FreeTrie *&root_found =
- FreeTrie::find(trie, block1->inner_size(), {0, WIDTH});
+ // Finding the root by its exact size.
+ FreeTrie *&root_found = FreeTrie::find(trie, block->inner_size(), {0, 1024});
EXPECT_EQ(&root_found, &trie);
- FreeTrie *&less_found = FreeTrie::find(trie, WIDTH / 2, {0, 1024});
- EXPECT_NE(&less_found, &trie);
- EXPECT_EQ(less_found, static_cast<FreeTrie *>(nullptr));
-
- FreeTrie *&greater_found = FreeTrie::find(trie, WIDTH / 2 + 1, {0, 1024});
- EXPECT_NE(&greater_found, &trie);
- EXPECT_NE(&greater_found, &less_found);
- EXPECT_EQ(greater_found, static_cast<FreeTrie *>(nullptr));
-}
+ // Sizes in the lower half of the range return one child.
+ FreeTrie *&lower_found = FreeTrie::find(trie, 1024 / 2, {0, 1024});
+ EXPECT_NE(&lower_found, &trie);
+ EXPECT_EQ(lower_found, static_cast<FreeTrie *>(nullptr));
-TEST(LlvmLibcFreeTrie, RootPopWithChild) {
- cpp::byte mem1[1024];
- optional<Block<> *> maybeBlock = Block<>::init(mem1);
- ASSERT_TRUE(maybeBlock.has_value());
- Block<> *block1 = *maybeBlock;
+ FreeTrie *&lower2_found = FreeTrie::find(trie, 0, {0, 1024});
+ EXPECT_EQ(&lower2_found, &lower_found);
- cpp::byte mem2[1024];
- maybeBlock = Block<>::init(mem2);
- ASSERT_TRUE(maybeBlock.has_value());
- Block<> *block2 = *maybeBlock;
+ // Sizes in the upper half of the range return the other child.
+ FreeTrie *&upper_found = FreeTrie::find(trie, 1024 / 2 + 1, {0, 1024});
+ EXPECT_NE(&upper_found, &trie);
+ EXPECT_NE(&upper_found, &lower_found);
+ EXPECT_EQ(upper_found, static_cast<FreeTrie *>(nullptr));
- cpp::byte mem3[2048];
- maybeBlock = Block<>::init(mem3);
- ASSERT_TRUE(maybeBlock.has_value());
- Block<> *block3 = *maybeBlock;
+ FreeTrie *&upper2_found = FreeTrie::find(trie, 1024 - 1, {0, 1024});
+ EXPECT_EQ(&upper2_found, &upper_found);
+}
- cpp::byte mem4[2047];
- maybeBlock = Block<>::init(mem4);
- ASSERT_TRUE(maybeBlock.has_value());
- Block<> *block4 = *maybeBlock;
+TEST(LlvmLibcFreeTrie, PopPreservesChildren) {
+ // Build the following trie:
+ // 1 -> 2
+ // lower:
+ // 3
+ // lower:
+ // 4
+ BlockMem<1024> block1_mem;
+ Block<> *block1 = block1_mem.block;
+ BlockMem<1024> block2_mem;
+ Block<> *block2 = block2_mem.block;
+ BlockMem<4096 / 2> block3_mem;
+ Block<> *block3 = block3_mem.block;
+ BlockMem<4096 / 2 - 1> block4_mem;
+ Block<> *block4 = block4_mem.block;
FreeTrie *trie = nullptr;
FreeTrie::push(trie, block1);
FreeTrie::push(trie, block2);
-
FreeTrie *&child3 = trie->find(trie, block3->inner_size(), {0, 4096});
FreeTrie::push(child3, block3);
-
- ASSERT_NE(child3, static_cast<FreeTrie *>(nullptr));
- EXPECT_EQ(child3->block(), block3);
-
FreeTrie *&child4 = trie->find(trie, block4->inner_size(), {0, 4096});
FreeTrie::push(child4, block4);
- ASSERT_NE(child4, static_cast<FreeTrie *>(nullptr));
- EXPECT_EQ(child4->block(), block4);
-
- // Expected Trie:
- // block1 -> block2
- // lower:
- // block3
- // upper:
- // block4
+ // Popping an element from the root preserves the child links.
FreeTrie::pop(trie);
FreeTrie *&new_child4 = trie->find(trie, block4->inner_size(), {0, 4096});
- // Expected Trie:
- // block2
- // lower:
- // block3
- // upper:
- // block4
EXPECT_EQ(new_child4, child4);
+ // Popping the last element from the root moves a leaf (block4) to the root
+ // and sets its children.
FreeTrie::pop(trie);
-
- // Expected Trie:
- // block4
- // lower:
- // block3
EXPECT_EQ(trie, child4);
FreeTrie *&new_child3 = trie->find(trie, block3->inner_size(), {0, 4096});
EXPECT_EQ(new_child3, child3);
>From 51a1fac98d2034e3ddca1f65e3585f1fe18aedc3 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Fri, 16 Aug 2024 13:39:57 -0700
Subject: [PATCH 22/54] More work on trie algorithm
---
libc/src/__support/freetrie.h | 60 +++++++++++++++++++++++------------
1 file changed, 39 insertions(+), 21 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 1608a03599e056..c7046796909dad 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -144,26 +144,36 @@ FreeTrie *&FreeTrie::find(FreeTrie *&trie, size_t size, SizeRange range) {
FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
SizeRange range) {
- if (!cur)
- return trie;
+ if (!trie)
+ return ≜
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
FreeTrie **cur = ≜
FreeTrie **best_fit = nullptr;
FreeTrie **deferred_upper_trie = nullptr;
+ SizeRange deferred_upper_range{0, 0};
// Inductively assume all better fits than the current best are in the
// current subtrie.
while (true) {
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
- // Consider whether the current node is a better fit.
- size_t cur_size = (*cur)->size();
- if (cur_size == size)
+ // If the current node is an exact fit, it is a best fit.
+ if ((*cur)->size() == size)
return cur;
- if (!best_fit || cur_size < (*best_fit)->size())
+
+ if (!best_fit || (*cur)->size() < (*best_fit)->size()) {
+ // The current node is a better fit.
best_fit = cur;
+ // The old deferred upper subtrie is outclassed by the new best fit.
+ LIBC_ASSERT(!deferred_upper_trie ||
+ deferred_upper_range.min > (*cur)->size() &&
+ "deferred upper subtrie should be an upper sibling of an "
+ "ancestor of the new best fit");
+ deferred_upper_trie = nullptr;
+ }
+
// Determine which subtries might contain better fits.
bool lower_impossible = !(*cur)->lower || range.lower().max() < size;
bool upper_impossible =
@@ -179,30 +189,38 @@ FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
cur = &(*cur)->lower;
range = range.lower();
} else {
- // Both subtries might contain a better fit. But, any fit in the lower
- // subtrie is better than the best fit in the upper subtrie. Accordingly,
- // we can scan the lower subtrie, then return to the upper one if
- // necessary.
+ // Both subtries might contain a better fit. Any fit in the lower subtrie
+ // is better than the any fit in the upper subtrie, so scan the lower
+ // subtrie and return to the upper one if necessary.
cur = &(*cur)->lower;
range = range.lower();
-
- // If we have already deferred a subtrie, it was an upper
- // subtrie of its parent, and the node being deferred here is somewhere in
- // the lower subtrie. That means that the new subtrie is strictly better
- // than the old, and the old can be summarily ignored.
+ LIBC_ASSERT(!deferred_upper_trie ||
+ range.upper().max() < deferred_upper_range.min &&
+ "old deferred upper subtrie should be an upper sibling "
+ "of an ancestor of the new deferred upper subtrie");
deferred_upper_trie = &(*cur)->upper;
}
}
- if (!deferred_upper_trie)
- return best_fit;
-
- // We have deferred an upper subtrie, and this is the only subtrie left that
- // could contain a better fit, via the above analysis. It's also the case that
- // any node within this subtrie is a fit, so just find the smallest.
+ // Scan the deferred upper subtrie and consider whether any element within
+ // provides a better fit.
cur = deferred_upper_trie;
+ range = deferred_upper_range;
while (*cur) {
+ if ((*cur)->size() < (*best_fit)->size())
+ best_fit = cur;
+ if ((*cur)->lower) {
+ cur = &(*cur)->lower;
+ range = range.lower();
+ } else {
+ if (best_fit && range.upper().min >= (*best_fit)->size())
+ break;
+ cur = &(*cur)->upper;
+ range = range.upper();
+ }
}
+
+ return best_fit;
}
FreeTrie &FreeTrie::leaf() {
>From 813d6d72c07b0e8f2aaa945c398469e478bffb1b Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Fri, 16 Aug 2024 14:43:13 -0700
Subject: [PATCH 23/54] Test find_best_fit
---
libc/src/__support/freetrie.h | 8 +++--
libc/test/src/__support/freetrie_test.cpp | 39 ++++++++++++-----------
2 files changed, 26 insertions(+), 21 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index c7046796909dad..85e79876ca2d9b 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -145,7 +145,7 @@ FreeTrie *&FreeTrie::find(FreeTrie *&trie, size_t size, SizeRange range) {
FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
SizeRange range) {
if (!trie)
- return ≜
+ return nullptr;
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
FreeTrie **cur = ≜
@@ -162,7 +162,8 @@ FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
if ((*cur)->size() == size)
return cur;
- if (!best_fit || (*cur)->size() < (*best_fit)->size()) {
+ if ((*cur)->size() > size &&
+ (!best_fit || (*cur)->size() < (*best_fit)->size())) {
// The current node is a better fit.
best_fit = cur;
@@ -202,6 +203,9 @@ FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
}
}
+ if (!deferred_upper_trie)
+ return best_fit;
+
// Scan the deferred upper subtrie and consider whether any element within
// provides a better fit.
cur = deferred_upper_trie;
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index b1487a88b99b38..c223ea3e4f3ad5 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -106,41 +106,42 @@ TEST(LlvmLibcFreeTrie, PopPreservesChildren) {
FreeTrie *trie = nullptr;
FreeTrie::push(trie, block1);
FreeTrie::push(trie, block2);
- FreeTrie *&child3 = trie->find(trie, block3->inner_size(), {0, 4096});
+ FreeTrie *&child3 = FreeTrie::find(trie, block3->inner_size(), {0, 4096});
FreeTrie::push(child3, block3);
- FreeTrie *&child4 = trie->find(trie, block4->inner_size(), {0, 4096});
+ FreeTrie *&child4 = FreeTrie::find(trie, block4->inner_size(), {0, 4096});
FreeTrie::push(child4, block4);
// Popping an element from the root preserves the child links.
FreeTrie::pop(trie);
- FreeTrie *&new_child4 = trie->find(trie, block4->inner_size(), {0, 4096});
+ FreeTrie *&new_child4 = FreeTrie::find(trie, block4->inner_size(), {0, 4096});
EXPECT_EQ(new_child4, child4);
// Popping the last element from the root moves a leaf (block4) to the root
// and sets its children.
FreeTrie::pop(trie);
EXPECT_EQ(trie, child4);
- FreeTrie *&new_child3 = trie->find(trie, block3->inner_size(), {0, 4096});
+ FreeTrie *&new_child3 = FreeTrie::find(trie, block3->inner_size(), {0, 4096});
EXPECT_EQ(new_child3, child3);
}
-TEST(LlvmLibcFreeTrie, SizeRange) {
- FreeTrie::SizeRange range(123, 1024);
- EXPECT_EQ(range.min, size_t{123});
- EXPECT_EQ(range.width, size_t{1024});
-
- EXPECT_TRUE(range.contains(123));
- EXPECT_TRUE(range.contains(123 + 1024 - 1));
- EXPECT_FALSE(range.contains(123 - 1));
- EXPECT_FALSE(range.contains(123 + 1024 + 1));
+TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
+ FreeTrie *trie = nullptr;
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, 123, {0, 4096}),
+ static_cast<FreeTrie **>(nullptr));
- FreeTrie::SizeRange lower = range.lower();
- EXPECT_EQ(lower.min, size_t{123});
- EXPECT_EQ(lower.width, size_t{1024 / 2});
+ BlockMem<1024> block_mem;
+ Block<> *block = block_mem.block;
+ FreeTrie::push(trie, block);
- FreeTrie::SizeRange upper = range.upper();
- EXPECT_EQ(upper.min, size_t{123 + 1024 / 2});
- EXPECT_EQ(upper.width, size_t{1024 / 2});
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, 0, {0, 4096}), &trie);
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, block->inner_size() - 1, {0, 4096}),
+ &trie);
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, block->inner_size(), {0, 4096}),
+ &trie);
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, block->inner_size() + 1, {0, 4096}),
+ static_cast<FreeTrie **>(nullptr));
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, 4096 - 1, {0, 4096}),
+ static_cast<FreeTrie **>(nullptr));
}
} // namespace LIBC_NAMESPACE_DECL
>From 6771365d3cb1c1c7ba9d012bb5fab3fda502bad6 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 20 Aug 2024 16:48:23 -0700
Subject: [PATCH 24/54] Merge the defer scan into the main loop
---
libc/config/linux/x86_64/entrypoints.txt | 1 +
libc/src/__support/freetrie.h | 40 +++++++++---------------
2 files changed, 16 insertions(+), 25 deletions(-)
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 2b76487218b13c..f18285cfad31c7 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -202,6 +202,7 @@ set(TARGET_LIBC_ENTRYPOINTS
libc.src.stdlib.aligned_alloc
libc.src.stdlib.calloc
libc.src.stdlib.free
+ libc.src.stdlib.freelist_malloc
libc.src.stdlib.malloc
libc.src.stdlib.realloc
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 85e79876ca2d9b..d5388da896f998 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -181,8 +181,21 @@ FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
!(*cur)->upper ||
(best_fit && range.upper().min >= (*best_fit)->size());
- if (lower_impossible && upper_impossible)
- break;
+ if (lower_impossible && upper_impossible) {
+ if (!deferred_upper_trie)
+ return best_fit;
+ // Scan the deferred upper subtrie and consider whether any element within
+ // provides a better fit.
+ //
+ // This can only ever be reached once. In a deferred upper subtrie, every
+ // node fits, so the scan can always summarily ignore an upper suptrie
+ // rather than deferring it.
+ cur = deferred_upper_trie;
+ range = deferred_upper_range;
+ deferred_upper_trie = nullptr;
+ continue;
+ }
+
if (lower_impossible) {
cur = &(*cur)->upper;
range = range.upper();
@@ -202,29 +215,6 @@ FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
deferred_upper_trie = &(*cur)->upper;
}
}
-
- if (!deferred_upper_trie)
- return best_fit;
-
- // Scan the deferred upper subtrie and consider whether any element within
- // provides a better fit.
- cur = deferred_upper_trie;
- range = deferred_upper_range;
- while (*cur) {
- if ((*cur)->size() < (*best_fit)->size())
- best_fit = cur;
- if ((*cur)->lower) {
- cur = &(*cur)->lower;
- range = range.lower();
- } else {
- if (best_fit && range.upper().min >= (*best_fit)->size())
- break;
- cur = &(*cur)->upper;
- range = range.upper();
- }
- }
-
- return best_fit;
}
FreeTrie &FreeTrie::leaf() {
>From 0686395a8350c298a53854b440be946e4b885047 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 20 Aug 2024 16:51:47 -0700
Subject: [PATCH 25/54] Minor cleanup
---
libc/src/__support/freetrie.h | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index d5388da896f998..6411a3fa6c397a 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -166,12 +166,10 @@ FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
(!best_fit || (*cur)->size() < (*best_fit)->size())) {
// The current node is a better fit.
best_fit = cur;
-
- // The old deferred upper subtrie is outclassed by the new best fit.
- LIBC_ASSERT(!deferred_upper_trie ||
- deferred_upper_range.min > (*cur)->size() &&
- "deferred upper subtrie should be an upper sibling of an "
- "ancestor of the new best fit");
+ LIBC_ASSERT(
+ !deferred_upper_trie ||
+ deferred_upper_range.min > (*cur)->size() &&
+ "deferred upper subtrie should be outclassed by new best fit");
deferred_upper_trie = nullptr;
}
@@ -210,8 +208,7 @@ FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
range = range.lower();
LIBC_ASSERT(!deferred_upper_trie ||
range.upper().max() < deferred_upper_range.min &&
- "old deferred upper subtrie should be an upper sibling "
- "of an ancestor of the new deferred upper subtrie");
+ "old deferred upper subtrie should be outclassed by new");
deferred_upper_trie = &(*cur)->upper;
}
}
>From f1c9f6b94e79192603b83ce9f485f8d5c2387dbb Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 21 Aug 2024 11:17:33 -0700
Subject: [PATCH 26/54] Lower only best fit test
---
libc/test/src/__support/freetrie_test.cpp | 39 ++++++++++++++++-------
1 file changed, 28 insertions(+), 11 deletions(-)
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index c223ea3e4f3ad5..80e91e42d9cf23 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -88,6 +88,8 @@ TEST(LlvmLibcFreeTrie, Find) {
}
TEST(LlvmLibcFreeTrie, PopPreservesChildren) {
+ FreeTrie::SizeRange range{0, 4096};
+
// Build the following trie:
// 1 -> 2
// lower:
@@ -106,42 +108,57 @@ TEST(LlvmLibcFreeTrie, PopPreservesChildren) {
FreeTrie *trie = nullptr;
FreeTrie::push(trie, block1);
FreeTrie::push(trie, block2);
- FreeTrie *&child3 = FreeTrie::find(trie, block3->inner_size(), {0, 4096});
+ FreeTrie *&child3 = FreeTrie::find(trie, block3->inner_size(), range);
FreeTrie::push(child3, block3);
- FreeTrie *&child4 = FreeTrie::find(trie, block4->inner_size(), {0, 4096});
+ FreeTrie *&child4 = FreeTrie::find(trie, block4->inner_size(), range);
FreeTrie::push(child4, block4);
// Popping an element from the root preserves the child links.
FreeTrie::pop(trie);
- FreeTrie *&new_child4 = FreeTrie::find(trie, block4->inner_size(), {0, 4096});
+ FreeTrie *&new_child4 = FreeTrie::find(trie, block4->inner_size(), range);
EXPECT_EQ(new_child4, child4);
// Popping the last element from the root moves a leaf (block4) to the root
// and sets its children.
FreeTrie::pop(trie);
EXPECT_EQ(trie, child4);
- FreeTrie *&new_child3 = FreeTrie::find(trie, block3->inner_size(), {0, 4096});
+ FreeTrie *&new_child3 = FreeTrie::find(trie, block3->inner_size(), range);
EXPECT_EQ(new_child3, child3);
}
TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
+ FreeTrie::SizeRange range{0, 4096};
+
FreeTrie *trie = nullptr;
- EXPECT_EQ(FreeTrie::find_best_fit(trie, 123, {0, 4096}),
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, 123, range),
static_cast<FreeTrie **>(nullptr));
BlockMem<1024> block_mem;
Block<> *block = block_mem.block;
FreeTrie::push(trie, block);
- EXPECT_EQ(FreeTrie::find_best_fit(trie, 0, {0, 4096}), &trie);
- EXPECT_EQ(FreeTrie::find_best_fit(trie, block->inner_size() - 1, {0, 4096}),
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, 0, range), &trie);
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, block->inner_size() - 1, range),
&trie);
- EXPECT_EQ(FreeTrie::find_best_fit(trie, block->inner_size(), {0, 4096}),
- &trie);
- EXPECT_EQ(FreeTrie::find_best_fit(trie, block->inner_size() + 1, {0, 4096}),
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, block->inner_size(), range), &trie);
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, block->inner_size() + 1, range),
static_cast<FreeTrie **>(nullptr));
- EXPECT_EQ(FreeTrie::find_best_fit(trie, 4096 - 1, {0, 4096}),
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, range.width - 1, range),
static_cast<FreeTrie **>(nullptr));
}
+TEST(LlvmLibcFreeTrie, FindBestFitLowerOnly) {
+ FreeTrie::SizeRange range{0, 4096};
+
+ FreeTrie *trie = nullptr;
+ BlockMem<1024> root_mem;
+ FreeTrie::push(trie, root_mem.block);
+ BlockMem<1024 - 1> lower_mem;
+ FreeTrie *&lower =
+ FreeTrie::find(trie, lower_mem.block->inner_size(), range);
+ FreeTrie::push(lower, lower_mem.block);
+
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, 0, range), &lower);
+}
+
} // namespace LIBC_NAMESPACE_DECL
>From 06120e80b34c6967443ae37c850ff324bf37f64c Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 21 Aug 2024 11:28:58 -0700
Subject: [PATCH 27/54] Upper only test
---
libc/src/__support/freetrie.h | 2 +-
libc/test/src/__support/freetrie_test.cpp | 24 ++++++++++++++++++++---
2 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 6411a3fa6c397a..88c80e1d09c5ed 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -156,7 +156,7 @@ FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
// Inductively assume all better fits than the current best are in the
// current subtrie.
while (true) {
- LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
+ LIBC_ASSERT(range.max() >= size && "range could not fit requested size");
// If the current node is an exact fit, it is a best fit.
if ((*cur)->size() == size)
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 80e91e42d9cf23..11784701d02102 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -126,7 +126,7 @@ TEST(LlvmLibcFreeTrie, PopPreservesChildren) {
EXPECT_EQ(new_child3, child3);
}
-TEST(LlvmLibcFreeTrie, FindBestFitRoot) {
+TEST(LlvmLibcFreeTrie, FindBestFitRootOnly) {
FreeTrie::SizeRange range{0, 4096};
FreeTrie *trie = nullptr;
@@ -154,11 +154,29 @@ TEST(LlvmLibcFreeTrie, FindBestFitLowerOnly) {
BlockMem<1024> root_mem;
FreeTrie::push(trie, root_mem.block);
BlockMem<1024 - 1> lower_mem;
- FreeTrie *&lower =
- FreeTrie::find(trie, lower_mem.block->inner_size(), range);
+ FreeTrie *&lower = FreeTrie::find(trie, lower_mem.block->inner_size(), range);
FreeTrie::push(lower, lower_mem.block);
EXPECT_EQ(FreeTrie::find_best_fit(trie, 0, range), &lower);
}
+TEST(LlvmLibcFreeTrie, FindBestFitUpperOnly) {
+ FreeTrie::SizeRange range{0, 4096};
+
+ FreeTrie *trie = nullptr;
+ BlockMem<1024> root_mem;
+ FreeTrie::push(trie, root_mem.block);
+ BlockMem<4096 - 1> upper_mem;
+ FreeTrie *&upper = FreeTrie::find(trie, upper_mem.block->inner_size(), range);
+ FreeTrie::push(upper, upper_mem.block);
+
+ EXPECT_EQ(
+ FreeTrie::find_best_fit(trie, root_mem.block->inner_size() + 1, range),
+ &upper);
+ // The upper subtrie should be skipped if it could not contain a better fit.
+ EXPECT_EQ(
+ FreeTrie::find_best_fit(trie, root_mem.block->inner_size() - 1, range),
+ &trie);
+}
+
} // namespace LIBC_NAMESPACE_DECL
>From 83f84989f878913dee22d359ca8542d2d1c15a98 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 21 Aug 2024 13:58:55 -0700
Subject: [PATCH 28/54] Lower and upper test
---
libc/src/__support/freetrie.h | 2 +-
libc/test/src/__support/freetrie_test.cpp | 29 ++++++++++++++++++++---
2 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 88c80e1d09c5ed..e489425bdc54ba 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -114,7 +114,7 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
FreeTrie &l = trie->leaf();
if (&l == trie) {
- // The last element of the trie was remved.
+ // If the root is a leaf, then removing it empties the trie.
trie = nullptr;
return;
}
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 11784701d02102..8c890179da7cf1 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -153,9 +153,10 @@ TEST(LlvmLibcFreeTrie, FindBestFitLowerOnly) {
FreeTrie *trie = nullptr;
BlockMem<1024> root_mem;
FreeTrie::push(trie, root_mem.block);
- BlockMem<1024 - 1> lower_mem;
- FreeTrie *&lower = FreeTrie::find(trie, lower_mem.block->inner_size(), range);
- FreeTrie::push(lower, lower_mem.block);
+ BlockMem<512> lower_mem;
+ Block<> *lower_block = lower_mem.block;
+ FreeTrie *&lower = FreeTrie::find(trie, lower_block->inner_size(), range);
+ FreeTrie::push(lower, lower_block);
EXPECT_EQ(FreeTrie::find_best_fit(trie, 0, range), &lower);
}
@@ -179,4 +180,26 @@ TEST(LlvmLibcFreeTrie, FindBestFitUpperOnly) {
&trie);
}
+TEST(LlvmLibcFreeTrie, FindBestFitLowerAndUpper) {
+ FreeTrie::SizeRange range{0, 4096};
+
+ FreeTrie *trie = nullptr;
+ BlockMem<1024> root_mem;
+ FreeTrie::push(trie, root_mem.block);
+ BlockMem<128> lower_mem;
+ FreeTrie *&lower = FreeTrie::find(trie, lower_mem.block->inner_size(), range);
+ FreeTrie::push(lower, lower_mem.block);
+ BlockMem<4096 - 1> upper_mem;
+ FreeTrie *&upper = FreeTrie::find(trie, upper_mem.block->inner_size(), range);
+ FreeTrie::push(upper, upper_mem.block);
+
+ // The lower subtrie is examined first.
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, 0, range), &lower);
+ // The upper subtrie is examined if there are no fits found in the upper
+ // subtrie.
+ EXPECT_EQ(
+ FreeTrie::find_best_fit(trie, lower_mem.block->inner_size() + 1, range),
+ &upper);
+}
+
} // namespace LIBC_NAMESPACE_DECL
>From d67abfc835da807d1b01451f67e8ae7c777e5479 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 21 Aug 2024 15:29:55 -0700
Subject: [PATCH 29/54] Test fix
---
libc/src/__support/freelist2.h | 1 +
libc/test/src/__support/freetrie_test.cpp | 4 +---
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist2.h
index 025a5df5dae0c9..5edb0eaa397e4c 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist2.h
@@ -47,6 +47,7 @@ class FreeList2 {
};
LIBC_INLINE void FreeList2::push(FreeList2 *&list, Block<> *block) {
+ LIBC_ASSERT(!block->used() && "only free blocks can be placed on free lists");
LIBC_ASSERT(block->inner_size_free() >= sizeof(FreeList2) &&
"block too small to accomodate free list node");
push(list, new (block->usable_space()) FreeList2);
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index 8c890179da7cf1..a3491f2ac382a5 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -197,9 +197,7 @@ TEST(LlvmLibcFreeTrie, FindBestFitLowerAndUpper) {
EXPECT_EQ(FreeTrie::find_best_fit(trie, 0, range), &lower);
// The upper subtrie is examined if there are no fits found in the upper
// subtrie.
- EXPECT_EQ(
- FreeTrie::find_best_fit(trie, lower_mem.block->inner_size() + 1, range),
- &upper);
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, range.width / 2, range), &upper);
}
} // namespace LIBC_NAMESPACE_DECL
>From 4dec829233f5a3baf9af30706ac35fb1efadfbb4 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 21 Aug 2024 16:35:22 -0700
Subject: [PATCH 30/54] Fill out freestore a bit
---
libc/src/__support/freestore.h | 57 ++++++++++++++++++++++
libc/test/src/__support/CMakeLists.txt | 1 +
libc/test/src/__support/freestore_test.cpp | 20 ++++++++
3 files changed, 78 insertions(+)
create mode 100644 libc/src/__support/freestore.h
create mode 100644 libc/test/src/__support/freestore_test.cpp
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
new file mode 100644
index 00000000000000..6d5c212c3000f0
--- /dev/null
+++ b/libc/src/__support/freestore.h
@@ -0,0 +1,57 @@
+//===-- Interface for freestore -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
+#define LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
+
+#include "freetrie.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LIBC_INLINE static constexpr size_t align_up(size_t value) {
+ constexpr size_t ALIGNMENT = alignof(max_align_t);
+ return (value + ALIGNMENT - 1) / ALIGNMENT * ALIGNMENT;
+}
+
+class FreeStore {
+public:
+ FreeStore(Block<> *block);
+
+private:
+ static constexpr size_t MIN_SIZE = sizeof(FreeList2);
+ static constexpr size_t MIN_LARGE_SIZE = sizeof(FreeTrie);
+ static constexpr size_t NUM_SMALL_SIZES =
+ (align_up(MIN_LARGE_SIZE) - align_up(MIN_SIZE)) / alignof(max_align_t);
+
+ static bool is_small(Block<> *block);
+
+ FreeList2 *&small_list(Block<> *block);
+
+ cpp::array<FreeList2 *, NUM_SMALL_SIZES> small_lists = {nullptr};
+ FreeTrie *large_trie = nullptr;
+};
+
+inline FreeStore::FreeStore(Block<> *block) {
+ if (is_small(block))
+ FreeList2::push(small_list(block), block);
+ else
+ FreeTrie::push(large_trie, block);
+}
+
+bool FreeStore::is_small(Block<> *block) {
+ return block->inner_size_free() <= MIN_LARGE_SIZE;
+}
+
+FreeList2 *&FreeStore::small_list(Block<> *block) {
+ LIBC_ASSERT(is_small(block) && "can legal for small blocks");
+ return small_lists[block->inner_size_free() / alignof(max_align_t)];
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index e012e3345afde4..e12a2499b7362a 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -40,6 +40,7 @@ if(LLVM_LIBC_FULL_BUILD)
fake_heap.s
freelist_heap_test.cpp
freelist_malloc_test.cpp
+ freestore_test.cpp
DEPENDS
libc.src.__support.CPP.span
libc.src.__support.freelist_heap
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
new file mode 100644
index 00000000000000..b593792d12ed42
--- /dev/null
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -0,0 +1,20 @@
+//===-- Unittests for a freestore -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <stddef.h>
+
+#include "src/__support/freestore.h"
+#include "test/UnitTest/Test.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+TEST(LlvmLibcFreeStore, PushPop) {
+ EXPECT_EQ(1, 2);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
>From 33e700498a62a14028d448448cfac16c8737c1b1 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 21 Aug 2024 16:36:12 -0700
Subject: [PATCH 31/54] Fix indexing
---
libc/src/__support/freestore.h | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 6d5c212c3000f0..4ea0e50f1dbd35 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -49,7 +49,8 @@ bool FreeStore::is_small(Block<> *block) {
FreeList2 *&FreeStore::small_list(Block<> *block) {
LIBC_ASSERT(is_small(block) && "can legal for small blocks");
- return small_lists[block->inner_size_free() / alignof(max_align_t)];
+ return small_lists[(block->inner_size_free() - MIN_SIZE) /
+ alignof(max_align_t)];
}
} // namespace LIBC_NAMESPACE_DECL
>From e1bc3f1b6eb67d35d5fa0e84cac4199882bc1c4b Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 21 Aug 2024 16:45:13 -0700
Subject: [PATCH 32/54] Rework constructor into 'add'
---
libc/src/__support/freestore.h | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 4ea0e50f1dbd35..c4e630a39e9f93 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -20,7 +20,9 @@ LIBC_INLINE static constexpr size_t align_up(size_t value) {
class FreeStore {
public:
- FreeStore(Block<> *block);
+ FreeStore(FreeTrie::SizeRange range) : range(range) {}
+
+ void add(Block<> *block);
private:
static constexpr size_t MIN_SIZE = sizeof(FreeList2);
@@ -34,20 +36,22 @@ class FreeStore {
cpp::array<FreeList2 *, NUM_SMALL_SIZES> small_lists = {nullptr};
FreeTrie *large_trie = nullptr;
+ FreeTrie::SizeRange range;
};
-inline FreeStore::FreeStore(Block<> *block) {
+inline void FreeStore::add(Block<> *block) {
if (is_small(block))
FreeList2::push(small_list(block), block);
else
- FreeTrie::push(large_trie, block);
+ FreeTrie::push(FreeTrie::find(large_trie, block->inner_size(), range),
+ block);
}
-bool FreeStore::is_small(Block<> *block) {
+inline bool FreeStore::is_small(Block<> *block) {
return block->inner_size_free() <= MIN_LARGE_SIZE;
}
-FreeList2 *&FreeStore::small_list(Block<> *block) {
+inline FreeList2 *&FreeStore::small_list(Block<> *block) {
LIBC_ASSERT(is_small(block) && "can legal for small blocks");
return small_lists[(block->inner_size_free() - MIN_SIZE) /
alignof(max_align_t)];
>From d8ef18177b368518d5568a49a0c4f5861f478301 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Thu, 22 Aug 2024 11:13:54 -0700
Subject: [PATCH 33/54] Remove best fit
---
libc/src/__support/freestore.h | 39 +++++++++++++++++++++++++++++-----
1 file changed, 34 insertions(+), 5 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index c4e630a39e9f93..db9e9f60c2d1c4 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -1,4 +1,4 @@
-//===-- Interface for freestore -------------------------------------------===//
+//===-- Interface for freestore ------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -22,7 +22,8 @@ class FreeStore {
public:
FreeStore(FreeTrie::SizeRange range) : range(range) {}
- void add(Block<> *block);
+ void insert(Block<> *block);
+ Block<> *remove_best_fit(size_t size);
private:
static constexpr size_t MIN_SIZE = sizeof(FreeList2);
@@ -31,15 +32,17 @@ class FreeStore {
(align_up(MIN_LARGE_SIZE) - align_up(MIN_SIZE)) / alignof(max_align_t);
static bool is_small(Block<> *block);
+ static bool is_small(size_t size);
FreeList2 *&small_list(Block<> *block);
+ FreeList2 *&small_list(size_t size);
cpp::array<FreeList2 *, NUM_SMALL_SIZES> small_lists = {nullptr};
FreeTrie *large_trie = nullptr;
FreeTrie::SizeRange range;
};
-inline void FreeStore::add(Block<> *block) {
+inline void FreeStore::insert(Block<> *block) {
if (is_small(block))
FreeList2::push(small_list(block), block);
else
@@ -47,12 +50,38 @@ inline void FreeStore::add(Block<> *block) {
block);
}
+inline Block<> *FreeStore::remove_best_fit(size_t size) {
+ if (is_small(size)) {
+ for (FreeList2 *&list : small_lists) {
+ if (!list || list->size() < size)
+ continue;
+ Block<> *block = list->block();
+ FreeList2::pop(list);
+ return block;
+ }
+ return nullptr;
+ } else {
+ FreeTrie **best_fit = FreeTrie::find_best_fit(large_trie, size, range);
+ if (!best_fit)
+ return nullptr;
+ Block<> *block = (*best_fit)->block();
+ FreeTrie::pop(*best_fit);
+ return block;
+ }
+}
+
inline bool FreeStore::is_small(Block<> *block) {
- return block->inner_size_free() <= MIN_LARGE_SIZE;
+ return block->inner_size_free() < MIN_LARGE_SIZE;
+}
+
+inline bool FreeStore::is_small(size_t size) {
+ if (size < sizeof(Block<>::offset_type))
+ return true;
+ return size - sizeof(Block<>::offset_type) < MIN_LARGE_SIZE;
}
inline FreeList2 *&FreeStore::small_list(Block<> *block) {
- LIBC_ASSERT(is_small(block) && "can legal for small blocks");
+ LIBC_ASSERT(is_small(block) && "only legal for small blocks");
return small_lists[(block->inner_size_free() - MIN_SIZE) /
alignof(max_align_t)];
}
>From 2eab41962e6dca9e4a65f2ab9500687af916db15 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Thu, 22 Aug 2024 15:38:06 -0700
Subject: [PATCH 34/54] Hack up alloc implementation
---
libc/src/__support/freelist_heap.h | 137 ++++++++----------
libc/src/__support/freestore.h | 12 +-
.../test/src/__support/freelist_heap_test.cpp | 20 +--
3 files changed, 76 insertions(+), 93 deletions(-)
diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h
index 6c860d039553ab..7636d431f6bc26 100644
--- a/libc/src/__support/freelist_heap.h
+++ b/libc/src/__support/freelist_heap.h
@@ -12,7 +12,7 @@
#include <stddef.h>
#include "block.h"
-#include "freelist.h"
+#include "freestore.h"
#include "src/__support/CPP/optional.h"
#include "src/__support/CPP/span.h"
#include "src/__support/libc_assert.h"
@@ -30,21 +30,14 @@ using cpp::span;
inline constexpr bool IsPow2(size_t x) { return x && (x & (x - 1)) == 0; }
-static constexpr cpp::array<size_t, 6> DEFAULT_BUCKETS{16, 32, 64,
- 128, 256, 512};
-
-template <size_t NUM_BUCKETS = DEFAULT_BUCKETS.size()> class FreeListHeap {
+class FreeListHeap {
public:
using BlockType = Block<>;
- using FreeListType = FreeList<NUM_BUCKETS>;
-
- static constexpr size_t MIN_ALIGNMENT =
- cpp::max(BlockType::ALIGNMENT, alignof(max_align_t));
- constexpr FreeListHeap() : begin_(&_end), end_(&__llvm_libc_heap_limit) {}
+ constexpr FreeListHeap() : begin(&_end), end(&__llvm_libc_heap_limit) {}
constexpr FreeListHeap(span<cpp::byte> region)
- : begin_(region.begin()), end_(region.end()) {}
+ : begin(region.begin()), end(region.end()) {}
void *allocate(size_t size);
void *aligned_allocate(size_t alignment, size_t size);
@@ -54,89 +47,75 @@ template <size_t NUM_BUCKETS = DEFAULT_BUCKETS.size()> class FreeListHeap {
void *realloc(void *ptr, size_t size);
void *calloc(size_t num, size_t size);
- cpp::span<cpp::byte> region() const { return {begin_, end_}; }
+ cpp::span<cpp::byte> region() const { return {begin, end}; }
private:
void init();
void *allocate_impl(size_t alignment, size_t size);
- span<cpp::byte> block_to_span(BlockType *block) {
+ span<cpp::byte> block_to_span(Block<> *block) {
return span<cpp::byte>(block->usable_space(), block->inner_size());
}
- bool is_valid_ptr(void *ptr) { return ptr >= begin_ && ptr < end_; }
+ bool is_valid_ptr(void *ptr) { return ptr >= begin && ptr < end; }
- bool is_initialized_ = false;
- cpp::byte *begin_;
- cpp::byte *end_;
- FreeListType freelist_{DEFAULT_BUCKETS};
+ cpp::byte *begin;
+ cpp::byte *end;
+ bool is_initialized = false;
+ FreeStore free_store;
};
-template <size_t BUFF_SIZE, size_t NUM_BUCKETS = DEFAULT_BUCKETS.size()>
-class FreeListHeapBuffer : public FreeListHeap<NUM_BUCKETS> {
- using parent = FreeListHeap<NUM_BUCKETS>;
- using FreeListNode = typename parent::FreeListType::FreeListNode;
-
+template <size_t BUFF_SIZE> class FreeListHeapBuffer : public FreeListHeap {
public:
- constexpr FreeListHeapBuffer()
- : FreeListHeap<NUM_BUCKETS>{buffer}, buffer{} {}
+ constexpr FreeListHeapBuffer() : FreeListHeap{buffer}, buffer{} {}
private:
cpp::byte buffer[BUFF_SIZE];
};
-template <size_t NUM_BUCKETS> void FreeListHeap<NUM_BUCKETS>::init() {
- LIBC_ASSERT(!is_initialized_ && "duplicate initialization");
- auto result = BlockType::init(region());
- BlockType *block = *result;
- freelist_.add_chunk(block_to_span(block));
- is_initialized_ = true;
+inline void FreeListHeap::init() {
+ LIBC_ASSERT(!is_initialized && "duplicate initialization");
+ auto result = Block<>::init(region());
+ Block<> *block = *result;
+ free_store.set_range({0, cpp::bit_ceil(block->inner_size())});
+ free_store.insert(block);
+ is_initialized = true;
}
-template <size_t NUM_BUCKETS>
-void *FreeListHeap<NUM_BUCKETS>::allocate_impl(size_t alignment, size_t size) {
+inline void *FreeListHeap::allocate_impl(size_t alignment, size_t size) {
if (size == 0)
return nullptr;
- if (!is_initialized_)
+ if (!is_initialized)
init();
- // Find a chunk in the freelist. Split it if needed, then return.
- auto chunk =
- freelist_.find_chunk_if([alignment, size](span<cpp::byte> chunk) {
- BlockType *block = BlockType::from_usable_space(chunk.data());
- return block->can_allocate(alignment, size);
- });
+ size_t request_size =
+ alignment <= alignof(max_align_t) ? size : size + alignment - 1;
- if (chunk.data() == nullptr)
+ Block<> *block = free_store.remove_best_fit(request_size);
+ if (!block)
return nullptr;
- freelist_.remove_chunk(chunk);
- BlockType *chunk_block = BlockType::from_usable_space(chunk.data());
- LIBC_ASSERT(!chunk_block->used());
+ LIBC_ASSERT(block->can_allocate(alignment, size) &&
+ "block should always be large enough to allocate at the correct "
+ "alignment");
- // Split that chunk. If there's a leftover chunk, add it to the freelist
- auto block_info = BlockType::allocate(chunk_block, alignment, size);
+ auto block_info = Block<>::allocate(block, alignment, size);
if (block_info.next)
- freelist_.add_chunk(block_to_span(block_info.next));
+ free_store.insert(block_info.next);
if (block_info.prev)
- freelist_.add_chunk(block_to_span(block_info.prev));
- chunk_block = block_info.block;
-
- chunk_block->mark_used();
+ free_store.insert(block_info.prev);
- return chunk_block->usable_space();
+ block_info.block->mark_used();
+ return block_info.block->usable_space();
}
-template <size_t NUM_BUCKETS>
-void *FreeListHeap<NUM_BUCKETS>::allocate(size_t size) {
- return allocate_impl(MIN_ALIGNMENT, size);
+inline void *FreeListHeap::allocate(size_t size) {
+ return allocate_impl(alignof(max_align_t), size);
}
-template <size_t NUM_BUCKETS>
-void *FreeListHeap<NUM_BUCKETS>::aligned_allocate(size_t alignment,
- size_t size) {
+inline void *FreeListHeap::aligned_allocate(size_t alignment, size_t size) {
// The alignment must be an integral power of two.
if (!IsPow2(alignment))
return nullptr;
@@ -148,38 +127,37 @@ void *FreeListHeap<NUM_BUCKETS>::aligned_allocate(size_t alignment,
return allocate_impl(alignment, size);
}
-template <size_t NUM_BUCKETS> void FreeListHeap<NUM_BUCKETS>::free(void *ptr) {
+inline void FreeListHeap::free(void *ptr) {
cpp::byte *bytes = static_cast<cpp::byte *>(ptr);
LIBC_ASSERT(is_valid_ptr(bytes) && "Invalid pointer");
- BlockType *chunk_block = BlockType::from_usable_space(bytes);
- LIBC_ASSERT(chunk_block->next() && "sentinel last block cannot be freed");
- LIBC_ASSERT(chunk_block->used() && "The block is not in-use");
- chunk_block->mark_free();
+ Block<> *block = Block<>::from_usable_space(bytes);
+ LIBC_ASSERT(block->next() && "sentinel last block cannot be freed");
+ LIBC_ASSERT(block->used() && "double free");
+ block->mark_free();
// Can we combine with the left or right blocks?
- BlockType *prev_free = chunk_block->prev_free();
- BlockType *next = chunk_block->next();
+ Block<> *prev_free = block->prev_free();
+ Block<> *next = block->next();
if (prev_free != nullptr) {
- // Remove from freelist and merge
- freelist_.remove_chunk(block_to_span(prev_free));
- chunk_block = prev_free;
- chunk_block->merge_next();
+ // Remove from free store and merge.
+ free_store.remove(prev_free);
+ block = prev_free;
+ block->merge_next();
}
if (!next->used()) {
- freelist_.remove_chunk(block_to_span(next));
- chunk_block->merge_next();
+ free_store.remove(next);
+ block->merge_next();
}
// Add back to the freelist
- freelist_.add_chunk(block_to_span(chunk_block));
+ free_store.insert(block);
}
// Follows constract of the C standard realloc() function
// If ptr is free'd, will return nullptr.
-template <size_t NUM_BUCKETS>
-void *FreeListHeap<NUM_BUCKETS>::realloc(void *ptr, size_t size) {
+inline void *FreeListHeap::realloc(void *ptr, size_t size) {
if (size == 0) {
free(ptr);
return nullptr;
@@ -194,10 +172,10 @@ void *FreeListHeap<NUM_BUCKETS>::realloc(void *ptr, size_t size) {
if (!is_valid_ptr(bytes))
return nullptr;
- BlockType *chunk_block = BlockType::from_usable_space(bytes);
- if (!chunk_block->used())
+ Block<> *block = Block<>::from_usable_space(bytes);
+ if (!block->used())
return nullptr;
- size_t old_size = chunk_block->inner_size();
+ size_t old_size = block->inner_size();
// Do nothing and return ptr if the required memory size is smaller than
// the current size.
@@ -214,15 +192,14 @@ void *FreeListHeap<NUM_BUCKETS>::realloc(void *ptr, size_t size) {
return new_ptr;
}
-template <size_t NUM_BUCKETS>
-void *FreeListHeap<NUM_BUCKETS>::calloc(size_t num, size_t size) {
+inline void *FreeListHeap::calloc(size_t num, size_t size) {
void *ptr = allocate(num * size);
if (ptr != nullptr)
LIBC_NAMESPACE::inline_memset(ptr, 0, num * size);
return ptr;
}
-extern FreeListHeap<> *freelist_heap;
+extern FreeListHeap *freelist_heap;
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index db9e9f60c2d1c4..328b398a887b43 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -20,8 +20,7 @@ LIBC_INLINE static constexpr size_t align_up(size_t value) {
class FreeStore {
public:
- FreeStore(FreeTrie::SizeRange range) : range(range) {}
-
+ void set_range(FreeTrie::SizeRange range);
void insert(Block<> *block);
Block<> *remove_best_fit(size_t size);
@@ -39,10 +38,17 @@ class FreeStore {
cpp::array<FreeList2 *, NUM_SMALL_SIZES> small_lists = {nullptr};
FreeTrie *large_trie = nullptr;
- FreeTrie::SizeRange range;
+ FreeTrie::SizeRange range = {0, 0};
};
+inline void FreeStore::set_range(FreeTrie::SizeRange range) {
+ LIBC_ASSERT(!large_trie && "cannot change the range of a preexisting trie");
+ this->range = range;
+}
+
inline void FreeStore::insert(Block<> *block) {
+ if (block->inner_size_free() < MIN_SIZE)
+ return;
if (is_small(block))
FreeList2::push(small_list(block), block);
else
diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp
index 973900dfdf56ea..b486fa5ac1bc3a 100644
--- a/libc/test/src/__support/freelist_heap_test.cpp
+++ b/libc/test/src/__support/freelist_heap_test.cpp
@@ -35,16 +35,16 @@ using LIBC_NAMESPACE::freelist_heap;
freelist_heap = \
new (&fake_global_buffer) FreeListHeapBuffer<BufferSize>; \
} \
- void RunTest(FreeListHeap<> &allocator, [[maybe_unused]] size_t N); \
+ void RunTest(FreeListHeap &allocator, [[maybe_unused]] size_t N); \
}; \
TEST_F(LlvmLibcFreeListHeapTest##TestCase, TestCase) { \
- alignas(FreeListHeap<>::BlockType) \
+ alignas(FreeListHeap::BlockType) \
cpp::byte buf[BufferSize] = {cpp::byte(0)}; \
- FreeListHeap<> allocator(buf); \
+ FreeListHeap allocator(buf); \
RunTest(allocator, BufferSize); \
RunTest(*freelist_heap, freelist_heap->region().size()); \
} \
- void LlvmLibcFreeListHeapTest##TestCase::RunTest(FreeListHeap<> &allocator, \
+ void LlvmLibcFreeListHeapTest##TestCase::RunTest(FreeListHeap &allocator, \
size_t N)
TEST_FOR_EACH_ALLOCATOR(CanAllocate, 2048) {
@@ -92,14 +92,14 @@ TEST_FOR_EACH_ALLOCATOR(ReturnsNullWhenAllocationTooLarge, 2048) {
// is used for other test cases and we don't explicitly free them.
TEST(LlvmLibcFreeListHeap, ReturnsNullWhenFull) {
constexpr size_t N = 2048;
- alignas(FreeListHeap<>::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
+ alignas(FreeListHeap::BlockType) cpp::byte buf[N] = {cpp::byte(0)};
- FreeListHeap<> allocator(buf);
+ FreeListHeap allocator(buf);
// Use aligned_allocate so we don't need to worry about ensuring the `buf`
// being aligned to max_align_t.
EXPECT_NE(allocator.aligned_allocate(
- 1, N - 2 * FreeListHeap<>::BlockType::BLOCK_OVERHEAD),
+ 1, N - 2 * FreeListHeap::BlockType::BLOCK_OVERHEAD),
static_cast<void *>(nullptr));
EXPECT_EQ(allocator.allocate(1), static_cast<void *>(nullptr));
}
@@ -246,12 +246,12 @@ TEST_FOR_EACH_ALLOCATOR(AlignedAlloc, 2048) {
// underlying buffer is not aligned to the alignments we request.
TEST(LlvmLibcFreeListHeap, AlignedAllocOnlyBlockTypeAligned) {
constexpr size_t BUFFER_SIZE = 4096;
- constexpr size_t BUFFER_ALIGNMENT = alignof(FreeListHeap<>::BlockType) * 2;
+ constexpr size_t BUFFER_ALIGNMENT = alignof(FreeListHeap::BlockType) * 2;
alignas(BUFFER_ALIGNMENT) cpp::byte buf[BUFFER_SIZE] = {cpp::byte(0)};
// Ensure the underlying buffer is at most aligned to the block type.
- FreeListHeap<> allocator(
- span<cpp::byte>(buf).subspan(alignof(FreeListHeap<>::BlockType)));
+ FreeListHeap allocator(
+ span<cpp::byte>(buf).subspan(alignof(FreeListHeap::BlockType)));
constexpr size_t ALIGNMENTS[] = {1, 2, 4, 8, 16, 32, 64, 128, 256};
constexpr size_t SIZE_SCALES[] = {1, 2, 3, 4, 5};
>From 4d44557a6fb5de3f6db95e8be71eab60b1800d08 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Fri, 23 Aug 2024 14:16:12 -0700
Subject: [PATCH 35/54] OOPS we need a parent link ohnooo
---
libc/src/__support/freestore.h | 44 +++++++++++++++++++++++-----------
1 file changed, 30 insertions(+), 14 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 328b398a887b43..90e0a3c65bdc71 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -22,6 +22,7 @@ class FreeStore {
public:
void set_range(FreeTrie::SizeRange range);
void insert(Block<> *block);
+ void remove(Block<> *block);
Block<> *remove_best_fit(size_t size);
private:
@@ -34,7 +35,7 @@ class FreeStore {
static bool is_small(size_t size);
FreeList2 *&small_list(Block<> *block);
- FreeList2 *&small_list(size_t size);
+ FreeList2 **best_small_fit(size_t size);
cpp::array<FreeList2 *, NUM_SMALL_SIZES> small_lists = {nullptr};
FreeTrie *large_trie = nullptr;
@@ -56,24 +57,32 @@ inline void FreeStore::insert(Block<> *block) {
block);
}
+inline void FreeStore::remove(Block<> *block) {
+ LIBC_ASSERT(block->inner_size_free() >= MIN_SIZE &&
+ "block too small to have been present");
+ if (is_small(block)) {
+ auto *list = static_cast<FreeList2 *>(block->usable_space());
+ } else {
+ auto *trie = static_cast<FreeTrie *>(block->usable_space());
+ }
+}
+
inline Block<> *FreeStore::remove_best_fit(size_t size) {
if (is_small(size)) {
- for (FreeList2 *&list : small_lists) {
- if (!list || list->size() < size)
- continue;
- Block<> *block = list->block();
- FreeList2::pop(list);
- return block;
- }
- return nullptr;
- } else {
- FreeTrie **best_fit = FreeTrie::find_best_fit(large_trie, size, range);
- if (!best_fit)
+ FreeList2 **list = best_small_fit(size);
+ if (!list)
return nullptr;
- Block<> *block = (*best_fit)->block();
- FreeTrie::pop(*best_fit);
+ Block<> *block = (*list)->block();
+ FreeList2::pop(*list);
return block;
}
+
+ FreeTrie **best_fit = FreeTrie::find_best_fit(large_trie, size, range);
+ if (!best_fit)
+ return nullptr;
+ Block<> *block = (*best_fit)->block();
+ FreeTrie::pop(*best_fit);
+ return block;
}
inline bool FreeStore::is_small(Block<> *block) {
@@ -92,6 +101,13 @@ inline FreeList2 *&FreeStore::small_list(Block<> *block) {
alignof(max_align_t)];
}
+inline FreeList2 **FreeStore::best_small_fit(size_t size) {
+ for (FreeList2 *&list : small_lists)
+ if (list && list->size() >= size)
+ return &list;
+ return nullptr;
+}
+
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC___SUPPORT_FREESTORE_H
>From 4a1c514c1a27f4f3f27b5bc96fdcf4b043bbb5fc Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Fri, 23 Aug 2024 14:20:56 -0700
Subject: [PATCH 36/54] It's okay for small lists though
---
libc/src/__support/freestore.h | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 90e0a3c65bdc71..d7224fd8faa6fe 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -61,7 +61,12 @@ inline void FreeStore::remove(Block<> *block) {
LIBC_ASSERT(block->inner_size_free() >= MIN_SIZE &&
"block too small to have been present");
if (is_small(block)) {
- auto *list = static_cast<FreeList2 *>(block->usable_space());
+ FreeList2 *block_list = static_cast<FreeList2 *>(block->usable_space());
+ FreeList2 *&list = small_list(block);
+ if (block_list == list)
+ FreeList2::pop(list);
+ else
+ FreeList2::pop(block_list);
} else {
auto *trie = static_cast<FreeTrie *>(block->usable_space());
}
>From e74e7026675f61c44ce62ee6985670cafdac6699 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Mon, 26 Aug 2024 10:51:11 -0700
Subject: [PATCH 37/54] Parent
---
libc/src/__support/freestore.h | 4 ++++
libc/src/__support/freetrie.h | 40 +++++++++++++++++++++++-----------
2 files changed, 31 insertions(+), 13 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index d7224fd8faa6fe..39c947f5b06df1 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -69,6 +69,10 @@ inline void FreeStore::remove(Block<> *block) {
FreeList2::pop(block_list);
} else {
auto *trie = static_cast<FreeTrie *>(block->usable_space());
+ if (trie == large_trie) {
+ FreeTrie2::pop(large_trie);
+ } else {
+ }
}
}
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index e489425bdc54ba..ae305db4d1e85c 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -1,4 +1,5 @@
-//===-- Interface for freetrie --------------------------------------------===//
+//===-- Interface for freetrie
+//--------------------------------------------===//freetrie.h
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -39,8 +40,13 @@ class FreeTrie : public FreeList2 {
bool contains(size_t size) const;
};
+ struct InsertPos {
+ FreeTrie *parent;
+ FreeTrie **trie;
+ };
+
/// Push to the back of this node's free list.
- static void push(FreeTrie *&trie, Block<> *block);
+ static void push(InsertPos pos, Block<> *block);
/// Pop from the front of this node's free list.
static void pop(FreeTrie *&trie);
@@ -48,7 +54,7 @@ class FreeTrie : public FreeList2 {
/// Finds the free trie for a given size. This may be a referance to a nullptr
/// at the correct place in the trie structure. The caller must provide the
/// SizeRange for this trie; the trie does not store it.
- static FreeTrie *&find(FreeTrie *&trie, size_t size, SizeRange range);
+ static InsertPos *&find(FreeTrie *&trie, size_t size, SizeRange range);
static FreeTrie **find_best_fit(FreeTrie *&trie, size_t size,
SizeRange range);
@@ -61,6 +67,8 @@ class FreeTrie : public FreeList2 {
FreeTrie *lower;
// The child subtrie covering the upper half of this subtrie's size range.
FreeTrie *upper;
+
+ FreeTrie *parent;
};
LIBC_INLINE FreeTrie::SizeRange::SizeRange(size_t min, size_t width)
@@ -86,16 +94,18 @@ LIBC_INLINE bool FreeTrie::SizeRange::contains(size_t size) const {
return true;
}
-LIBC_INLINE void FreeTrie::push(FreeTrie *&trie, Block<> *block) {
+LIBC_INLINE void FreeTrie::push(InsertPos pos, Block<> *block) {
LIBC_ASSERT(block->inner_size_free() >= sizeof(FreeTrie) &&
"block too small to accomodate free trie node");
FreeTrie *node = new (block->usable_space()) FreeTrie;
// The trie links are irrelevant for all but the first node in the free list.
- if (!trie)
+ if (!*pos.trie) {
node->lower = node->upper = nullptr;
- FreeList2 *list = trie;
+ node->parent = pos.parent;
+ }
+ FreeList2 *list = *pos.trie;
FreeList2::push(list, node);
- trie = static_cast<FreeTrie *>(list);
+ *pos.trie = static_cast<FreeTrie *>(list);
}
LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
@@ -106,6 +116,7 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
// The freelist is non-empty, so copy the trie links to the new head.
new_trie->lower = trie->lower;
new_trie->upper = trie->upper;
+ new_trie->parent = trie->parent;
trie = new_trie;
return;
}
@@ -123,23 +134,26 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
// no relationship between the size of the root and its children.
l.lower = trie->lower;
l.upper = trie->upper;
+ l.parent = trie->parent;
trie = &l;
}
-FreeTrie *&FreeTrie::find(FreeTrie *&trie, size_t size, SizeRange range) {
+FreeTrie::InsertPos FreeTrie::find(FreeTrie *&trie, size_t size,
+ SizeRange range) {
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
- FreeTrie **cur = ≜
- while (*cur && (*cur)->size() != size) {
+ InsertPos pos = {nullptr, &trie};
+ while (*pos.trie && (*pos.trie)->size() != size) {
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
+ pos.parent = *pos.trie;
if (range.lower().contains(size)) {
- cur = &(*cur)->lower;
+ pos.trie = &(*pos.trie)->lower;
range = range.lower();
} else {
- cur = &(*cur)->upper;
+ pos.trie = &(*pos.trie)->upper;
range = range.upper();
}
}
- return *cur;
+ return pos;
}
FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
>From 3917b0158f6e8e2ef94a375638009b1f6f0260c2 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Mon, 26 Aug 2024 10:53:59 -0700
Subject: [PATCH 38/54] Disable free trie test
---
libc/test/src/__support/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index e12a2499b7362a..0e74e81380c2ba 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -22,7 +22,7 @@ if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
SRCS
freelist_test.cpp
freelist2_test.cpp
- freetrie_test.cpp
+ #freetrie_test.cpp
DEPENDS
libc.src.__support.CPP.array
libc.src.__support.CPP.span
>From 865712c794eb34df40b32a23fa690ead45dc97d8 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Mon, 26 Aug 2024 10:54:12 -0700
Subject: [PATCH 39/54] Fix return type
---
libc/src/__support/freetrie.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index ae305db4d1e85c..b1ce04d68f2c9b 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -54,7 +54,7 @@ class FreeTrie : public FreeList2 {
/// Finds the free trie for a given size. This may be a referance to a nullptr
/// at the correct place in the trie structure. The caller must provide the
/// SizeRange for this trie; the trie does not store it.
- static InsertPos *&find(FreeTrie *&trie, size_t size, SizeRange range);
+ static InsertPos find(FreeTrie *&trie, size_t size, SizeRange range);
static FreeTrie **find_best_fit(FreeTrie *&trie, size_t size,
SizeRange range);
>From f60de08cb4d9ce673997f9183dfcf5fea526355f Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Mon, 26 Aug 2024 11:00:46 -0700
Subject: [PATCH 40/54] Clever self
---
libc/src/__support/freestore.h | 8 +++++---
libc/src/__support/freetrie.h | 7 +++++++
2 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 39c947f5b06df1..1d71055a521dd8 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -61,17 +61,19 @@ inline void FreeStore::remove(Block<> *block) {
LIBC_ASSERT(block->inner_size_free() >= MIN_SIZE &&
"block too small to have been present");
if (is_small(block)) {
- FreeList2 *block_list = static_cast<FreeList2 *>(block->usable_space());
+ FreeList2 *block_list =
+ reinterpret_cast<FreeList2 *>(block->usable_space());
FreeList2 *&list = small_list(block);
if (block_list == list)
FreeList2::pop(list);
else
FreeList2::pop(block_list);
} else {
- auto *trie = static_cast<FreeTrie *>(block->usable_space());
+ auto *trie = reinterpret_cast<FreeTrie *>(block->usable_space());
if (trie == large_trie) {
- FreeTrie2::pop(large_trie);
+ FreeTrie::pop(large_trie);
} else {
+ FreeTrie::pop(trie->self());
}
}
}
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index b1ce04d68f2c9b..8210a4734be448 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -45,6 +45,8 @@ class FreeTrie : public FreeList2 {
FreeTrie **trie;
};
+ FreeTrie *&self();
+
/// Push to the back of this node's free list.
static void push(InsertPos pos, Block<> *block);
@@ -71,6 +73,11 @@ class FreeTrie : public FreeList2 {
FreeTrie *parent;
};
+inline FreeTrie *&FreeTrie::self() {
+ LIBC_ASSERT(parent && "reference in parent not defined on root");
+ return parent->lower == this ? parent->lower : parent->upper;
+}
+
LIBC_INLINE FreeTrie::SizeRange::SizeRange(size_t min, size_t width)
: min(min), width(width) {
LIBC_ASSERT(!(width & (width - 1)) && "width must be a power of two");
>From ca4a1da3066cbf5d021a20af712e840b0f1a5cc1 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Mon, 26 Aug 2024 11:32:27 -0700
Subject: [PATCH 41/54] Fix uses of freelistheap; other fixes
---
libc/src/__support/freetrie.h | 18 ++++++++++--------
libc/src/stdlib/freelist_malloc.cpp | 4 ++--
.../src/__support/freelist_malloc_test.cpp | 2 +-
3 files changed, 13 insertions(+), 11 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 8210a4734be448..8e0fb0f2d7495c 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -24,7 +24,7 @@ class FreeTrie : public FreeList2 {
size_t min;
size_t width;
- SizeRange(size_t min, size_t width);
+ constexpr SizeRange(size_t min, size_t width);
/// @returns The lower half of the size range.
SizeRange lower() const;
@@ -78,7 +78,7 @@ inline FreeTrie *&FreeTrie::self() {
return parent->lower == this ? parent->lower : parent->upper;
}
-LIBC_INLINE FreeTrie::SizeRange::SizeRange(size_t min, size_t width)
+LIBC_INLINE constexpr FreeTrie::SizeRange::SizeRange(size_t min, size_t width)
: min(min), width(width) {
LIBC_ASSERT(!(width & (width - 1)) && "width must be a power of two");
}
@@ -91,7 +91,9 @@ LIBC_INLINE FreeTrie::SizeRange FreeTrie::SizeRange::upper() const {
return {min + width / 2, width / 2};
}
-size_t FreeTrie::SizeRange::max() const { return min + (width - 1); }
+LIBC_INLINE size_t FreeTrie::SizeRange::max() const {
+ return min + (width - 1);
+}
LIBC_INLINE bool FreeTrie::SizeRange::contains(size_t size) const {
if (size < min)
@@ -145,8 +147,8 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
trie = &l;
}
-FreeTrie::InsertPos FreeTrie::find(FreeTrie *&trie, size_t size,
- SizeRange range) {
+LIBC_INLINE FreeTrie::InsertPos FreeTrie::find(FreeTrie *&trie, size_t size,
+ SizeRange range) {
LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
InsertPos pos = {nullptr, &trie};
while (*pos.trie && (*pos.trie)->size() != size) {
@@ -163,8 +165,8 @@ FreeTrie::InsertPos FreeTrie::find(FreeTrie *&trie, size_t size,
return pos;
}
-FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
- SizeRange range) {
+LIBC_INLINE FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
+ SizeRange range) {
if (!trie)
return nullptr;
@@ -235,7 +237,7 @@ FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
}
}
-FreeTrie &FreeTrie::leaf() {
+LIBC_INLINE FreeTrie &FreeTrie::leaf() {
FreeTrie *cur = this;
while (cur->lower || cur->upper)
cur = cur->lower ? cur->lower : cur->upper;
diff --git a/libc/src/stdlib/freelist_malloc.cpp b/libc/src/stdlib/freelist_malloc.cpp
index 47240bc53aa37b..fe56fad769378a 100644
--- a/libc/src/stdlib/freelist_malloc.cpp
+++ b/libc/src/stdlib/freelist_malloc.cpp
@@ -18,8 +18,8 @@
namespace LIBC_NAMESPACE_DECL {
-static LIBC_CONSTINIT FreeListHeap<> freelist_heap_symbols;
-FreeListHeap<> *freelist_heap = &freelist_heap_symbols;
+static LIBC_CONSTINIT FreeListHeap freelist_heap_symbols;
+FreeListHeap *freelist_heap = &freelist_heap_symbols;
LLVM_LIBC_FUNCTION(void *, malloc, (size_t size)) {
return freelist_heap->allocate(size);
diff --git a/libc/test/src/__support/freelist_malloc_test.cpp b/libc/test/src/__support/freelist_malloc_test.cpp
index 9cbdec89f6576e..5372ec6fe253c0 100644
--- a/libc/test/src/__support/freelist_malloc_test.cpp
+++ b/libc/test/src/__support/freelist_malloc_test.cpp
@@ -22,7 +22,7 @@ TEST(LlvmLibcFreeListMalloc, Malloc) {
constexpr size_t kCallocNum = 4;
constexpr size_t kCallocSize = 64;
- typedef FreeListHeap<>::BlockType Block;
+ typedef FreeListHeap::BlockType Block;
void *ptr1 = LIBC_NAMESPACE::malloc(kAllocSize);
auto *block = Block::from_usable_space(ptr1);
>From 18e5b4c21af3b898d5ae77d4d892562f399b34a7 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Mon, 26 Aug 2024 12:40:57 -0700
Subject: [PATCH 42/54] Delete freelist
---
libc/src/__support/CMakeLists.txt | 2 +-
libc/src/__support/freelist.h | 209 ----------------------
libc/test/src/__support/CMakeLists.txt | 1 -
libc/test/src/__support/freelist_test.cpp | 166 -----------------
4 files changed, 1 insertion(+), 377 deletions(-)
delete mode 100644 libc/src/__support/freelist.h
delete mode 100644 libc/test/src/__support/freelist_test.cpp
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index d8a192f1ffa570..e98c51b56ff03f 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -17,7 +17,7 @@ add_header_library(
add_header_library(
freelist
HDRS
- freelist.h
+ freelist2.h
DEPENDS
libc.src.__support.fixedvector
libc.src.__support.CPP.array
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
deleted file mode 100644
index a54cf953fe7ab6..00000000000000
--- a/libc/src/__support/freelist.h
+++ /dev/null
@@ -1,209 +0,0 @@
-//===-- Interface for freelist_malloc -------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST_H
-#define LLVM_LIBC_SRC___SUPPORT_FREELIST_H
-
-#include "src/__support/CPP/array.h"
-#include "src/__support/CPP/cstddef.h"
-#include "src/__support/CPP/new.h"
-#include "src/__support/CPP/span.h"
-#include "src/__support/fixedvector.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-using cpp::span;
-
-/// Basic [freelist](https://en.wikipedia.org/wiki/Free_list) implementation
-/// for an allocator. This implementation buckets by chunk size, with a list
-/// of user-provided buckets. Each bucket is a linked list of storage chunks.
-/// Because this freelist uses the added chunks themselves as list nodes, there
-/// is a lower bound of `sizeof(FreeList.FreeListNode)` bytes for chunks which
-/// can be added to this freelist. There is also an implicit bucket for
-/// "everything else", for chunks which do not fit into a bucket.
-///
-/// Each added chunk will be added to the smallest bucket under which it fits.
-/// If it does not fit into any user-provided bucket, it will be added to the
-/// default bucket.
-///
-/// As an example, assume that the `FreeList` is configured with buckets of
-/// sizes {64, 128, 256, and 512} bytes. The internal state may look like the
-/// following:
-///
-/// @code{.unparsed}
-/// bucket[0] (64B) --> chunk[12B] --> chunk[42B] --> chunk[64B] --> NULL
-/// bucket[1] (128B) --> chunk[65B] --> chunk[72B] --> NULL
-/// bucket[2] (256B) --> NULL
-/// bucket[3] (512B) --> chunk[312B] --> chunk[512B] --> chunk[416B] --> NULL
-/// bucket[4] (implicit) --> chunk[1024B] --> chunk[513B] --> NULL
-/// @endcode
-///
-/// Note that added chunks should be aligned to a 4-byte boundary.
-template <size_t NUM_BUCKETS = 6> class FreeList {
-public:
- // Remove copy/move ctors
- FreeList(const FreeList &other) = delete;
- FreeList(FreeList &&other) = delete;
- FreeList &operator=(const FreeList &other) = delete;
- FreeList &operator=(FreeList &&other) = delete;
-
- /// Adds a chunk to this freelist.
- bool add_chunk(cpp::span<cpp::byte> chunk);
-
- /// Finds an eligible chunk for an allocation of size `size`.
- ///
- /// @note This returns the first allocation possible within a given bucket;
- /// It does not currently optimize for finding the smallest chunk.
- ///
- /// @returns
- /// * On success - A span representing the chunk.
- /// * On failure (e.g. there were no chunks available for that allocation) -
- /// A span with a size of 0.
- cpp::span<cpp::byte> find_chunk(size_t size) const;
-
- template <typename Cond> cpp::span<cpp::byte> find_chunk_if(Cond op) const;
-
- /// Removes a chunk from this freelist.
- bool remove_chunk(cpp::span<cpp::byte> chunk);
-
- /// For a given size, find which index into chunks_ the node should be written
- /// to.
- constexpr size_t find_chunk_ptr_for_size(size_t size, bool non_null) const;
-
- struct FreeListNode {
- FreeListNode *next;
- size_t size;
- };
-
- constexpr void set_freelist_node(FreeListNode &node,
- cpp::span<cpp::byte> chunk);
-
- constexpr explicit FreeList(const cpp::array<size_t, NUM_BUCKETS> &sizes)
- : chunks_(NUM_BUCKETS + 1, 0), sizes_(sizes.begin(), sizes.end()) {}
-
-private:
- FixedVector<FreeList::FreeListNode *, NUM_BUCKETS + 1> chunks_;
- FixedVector<size_t, NUM_BUCKETS> sizes_;
-};
-
-template <size_t NUM_BUCKETS>
-constexpr void FreeList<NUM_BUCKETS>::set_freelist_node(FreeListNode &node,
- span<cpp::byte> chunk) {
- // Add it to the correct list.
- size_t chunk_ptr = find_chunk_ptr_for_size(chunk.size(), false);
- node.size = chunk.size();
- node.next = chunks_[chunk_ptr];
- chunks_[chunk_ptr] = &node;
-}
-
-template <size_t NUM_BUCKETS>
-bool FreeList<NUM_BUCKETS>::add_chunk(span<cpp::byte> chunk) {
- // Check that the size is enough to actually store what we need
- if (chunk.size() < sizeof(FreeListNode))
- return false;
-
- FreeListNode *node = ::new (chunk.data()) FreeListNode;
- set_freelist_node(*node, chunk);
-
- return true;
-}
-
-template <size_t NUM_BUCKETS>
-template <typename Cond>
-span<cpp::byte> FreeList<NUM_BUCKETS>::find_chunk_if(Cond op) const {
- for (FreeListNode *node : chunks_) {
- while (node != nullptr) {
- span<cpp::byte> chunk(reinterpret_cast<cpp::byte *>(node), node->size);
- if (op(chunk))
- return chunk;
-
- node = node->next;
- }
- }
-
- return {};
-}
-
-template <size_t NUM_BUCKETS>
-span<cpp::byte> FreeList<NUM_BUCKETS>::find_chunk(size_t size) const {
- if (size == 0)
- return span<cpp::byte>();
-
- size_t chunk_ptr = find_chunk_ptr_for_size(size, true);
-
- // Check that there's data. This catches the case where we run off the
- // end of the array
- if (chunks_[chunk_ptr] == nullptr)
- return span<cpp::byte>();
-
- // Now iterate up the buckets, walking each list to find a good candidate
- for (size_t i = chunk_ptr; i < chunks_.size(); i++) {
- FreeListNode *node = chunks_[static_cast<unsigned short>(i)];
-
- while (node != nullptr) {
- if (node->size >= size)
- return span<cpp::byte>(reinterpret_cast<cpp::byte *>(node), node->size);
-
- node = node->next;
- }
- }
-
- // If we get here, we've checked every block in every bucket. There's
- // nothing that can support this allocation.
- return span<cpp::byte>();
-}
-
-template <size_t NUM_BUCKETS>
-bool FreeList<NUM_BUCKETS>::remove_chunk(span<cpp::byte> chunk) {
- size_t chunk_ptr = find_chunk_ptr_for_size(chunk.size(), true);
-
- // Check head first.
- if (chunks_[chunk_ptr] == nullptr)
- return false;
-
- FreeListNode *node = chunks_[chunk_ptr];
- if (reinterpret_cast<cpp::byte *>(node) == chunk.data()) {
- chunks_[chunk_ptr] = node->next;
- return true;
- }
-
- // No? Walk the nodes.
- node = chunks_[chunk_ptr];
-
- while (node->next != nullptr) {
- if (reinterpret_cast<cpp::byte *>(node->next) == chunk.data()) {
- // Found it, remove this node out of the chain
- node->next = node->next->next;
- return true;
- }
-
- node = node->next;
- }
-
- return false;
-}
-
-template <size_t NUM_BUCKETS>
-constexpr size_t
-FreeList<NUM_BUCKETS>::find_chunk_ptr_for_size(size_t size,
- bool non_null) const {
- size_t chunk_ptr = 0;
- for (chunk_ptr = 0u; chunk_ptr < sizes_.size(); chunk_ptr++) {
- if (sizes_[chunk_ptr] >= size &&
- (!non_null || chunks_[chunk_ptr] != nullptr)) {
- break;
- }
- }
-
- return chunk_ptr;
-}
-
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC___SUPPORT_FREELIST_H
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 0e74e81380c2ba..1bda590b55e102 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -20,7 +20,6 @@ if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
SUITE
libc-support-tests
SRCS
- freelist_test.cpp
freelist2_test.cpp
#freetrie_test.cpp
DEPENDS
diff --git a/libc/test/src/__support/freelist_test.cpp b/libc/test/src/__support/freelist_test.cpp
deleted file mode 100644
index cae0ed470315c1..00000000000000
--- a/libc/test/src/__support/freelist_test.cpp
+++ /dev/null
@@ -1,166 +0,0 @@
-//===-- Unittests for a freelist --------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include <stddef.h>
-
-#include "src/__support/CPP/array.h"
-#include "src/__support/CPP/span.h"
-#include "src/__support/freelist.h"
-#include "test/UnitTest/Test.h"
-
-using LIBC_NAMESPACE::FreeList;
-using LIBC_NAMESPACE::cpp::array;
-using LIBC_NAMESPACE::cpp::byte;
-using LIBC_NAMESPACE::cpp::span;
-
-static constexpr size_t SIZE = 8;
-static constexpr array<size_t, SIZE> example_sizes = {64, 128, 256, 512,
- 1024, 2048, 4096, 8192};
-
-TEST(LlvmLibcFreeList, EmptyListHasNoMembers) {
- FreeList<SIZE> list(example_sizes);
-
- auto item = list.find_chunk(4);
- EXPECT_EQ(item.size(), static_cast<size_t>(0));
- item = list.find_chunk(128);
- EXPECT_EQ(item.size(), static_cast<size_t>(0));
-}
-
-TEST(LlvmLibcFreeList, CanRetrieveAddedMember) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t N = 512;
-
- byte data[N] = {byte(0)};
-
- bool ok = list.add_chunk(span<byte>(data, N));
- EXPECT_TRUE(ok);
-
- auto item = list.find_chunk(N);
- EXPECT_EQ(item.size(), N);
- EXPECT_EQ(item.data(), data);
-}
-
-TEST(LlvmLibcFreeList, CanRetrieveAddedMemberForSmallerSize) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t N = 512;
-
- byte data[N] = {byte(0)};
-
- ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
- auto item = list.find_chunk(N / 2);
- EXPECT_EQ(item.size(), N);
- EXPECT_EQ(item.data(), data);
-}
-
-TEST(LlvmLibcFreeList, CanRemoveItem) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t N = 512;
-
- byte data[N] = {byte(0)};
-
- ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
- EXPECT_TRUE(list.remove_chunk(span<byte>(data, N)));
-
- auto item = list.find_chunk(N);
- EXPECT_EQ(item.size(), static_cast<size_t>(0));
-}
-
-TEST(LlvmLibcFreeList, FindReturnsSmallestChunk) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t kN1 = 512;
- constexpr size_t kN2 = 1024;
-
- byte data1[kN1] = {byte(0)};
- byte data2[kN2] = {byte(0)};
-
- ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
- ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));
-
- auto chunk = list.find_chunk(kN1 / 2);
- EXPECT_EQ(chunk.size(), kN1);
- EXPECT_EQ(chunk.data(), data1);
-
- chunk = list.find_chunk(kN1);
- EXPECT_EQ(chunk.size(), kN1);
- EXPECT_EQ(chunk.data(), data1);
-
- chunk = list.find_chunk(kN1 + 1);
- EXPECT_EQ(chunk.size(), kN2);
- EXPECT_EQ(chunk.data(), data2);
-}
-
-TEST(LlvmLibcFreeList, FindReturnsCorrectChunkInSameBucket) {
- // If we have two values in the same bucket, ensure that the allocation will
- // pick an appropriately sized one.
- FreeList<SIZE> list(example_sizes);
- constexpr size_t kN1 = 512;
- constexpr size_t kN2 = 257;
-
- byte data1[kN1] = {byte(0)};
- byte data2[kN2] = {byte(0)};
-
- // List should now be 257 -> 512 -> NULL
- ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
- ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));
-
- auto chunk = list.find_chunk(kN2 + 1);
- EXPECT_EQ(chunk.size(), kN1);
-}
-
-TEST(LlvmLibcFreeList, FindCanMoveUpThroughBuckets) {
- // Ensure that finding a chunk will move up through buckets if no appropriate
- // chunks were found in a given bucket
- FreeList<SIZE> list(example_sizes);
- constexpr size_t kN1 = 257;
- constexpr size_t kN2 = 513;
-
- byte data1[kN1] = {byte(0)};
- byte data2[kN2] = {byte(0)};
-
- // List should now be:
- // bkt[3] (257 bytes up to 512 bytes) -> 257 -> NULL
- // bkt[4] (513 bytes up to 1024 bytes) -> 513 -> NULL
- ASSERT_TRUE(list.add_chunk(span<byte>(data1, kN1)));
- ASSERT_TRUE(list.add_chunk(span<byte>(data2, kN2)));
-
- // Request a 300 byte chunk. This should return the 513 byte one
- auto chunk = list.find_chunk(kN1 + 1);
- EXPECT_EQ(chunk.size(), kN2);
-}
-
-TEST(LlvmLibcFreeList, RemoveUnknownChunkReturnsNotFound) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t N = 512;
-
- byte data[N] = {byte(0)};
- byte data2[N] = {byte(0)};
-
- ASSERT_TRUE(list.add_chunk(span<byte>(data, N)));
- EXPECT_FALSE(list.remove_chunk(span<byte>(data2, N)));
-}
-
-TEST(LlvmLibcFreeList, CanStoreMultipleChunksPerBucket) {
- FreeList<SIZE> list(example_sizes);
- constexpr size_t N = 512;
-
- byte data1[N] = {byte(0)};
- byte data2[N] = {byte(0)};
-
- ASSERT_TRUE(list.add_chunk(span<byte>(data1, N)));
- ASSERT_TRUE(list.add_chunk(span<byte>(data2, N)));
-
- auto chunk1 = list.find_chunk(N);
- ASSERT_TRUE(list.remove_chunk(chunk1));
- auto chunk2 = list.find_chunk(N);
- ASSERT_TRUE(list.remove_chunk(chunk2));
-
- // Ordering of the chunks doesn't matter
- EXPECT_TRUE(chunk1.data() != chunk2.data());
- EXPECT_TRUE(chunk1.data() == data1 || chunk1.data() == data2);
- EXPECT_TRUE(chunk2.data() == data1 || chunk2.data() == data2);
-}
>From cbab5b2ef793298663928a75b9c7dabb9c47e39a Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 27 Aug 2024 09:52:18 -0700
Subject: [PATCH 43/54] Rename freelist2->freelist
---
libc/src/__support/CMakeLists.txt | 2 +-
.../src/__support/{freelist2.h => freelist.h} | 32 +++++++++----------
libc/src/__support/freestore.h | 30 ++++++++---------
libc/src/__support/freetrie.h | 12 +++----
libc/test/src/__support/CMakeLists.txt | 2 +-
.../{freelist2_test.cpp => freelist_test.cpp} | 20 ++++++------
6 files changed, 49 insertions(+), 49 deletions(-)
rename libc/src/__support/{freelist2.h => freelist.h} (72%)
rename libc/test/src/__support/{freelist2_test.cpp => freelist_test.cpp} (70%)
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index e98c51b56ff03f..d8a192f1ffa570 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -17,7 +17,7 @@ add_header_library(
add_header_library(
freelist
HDRS
- freelist2.h
+ freelist.h
DEPENDS
libc.src.__support.fixedvector
libc.src.__support.CPP.array
diff --git a/libc/src/__support/freelist2.h b/libc/src/__support/freelist.h
similarity index 72%
rename from libc/src/__support/freelist2.h
rename to libc/src/__support/freelist.h
index 5edb0eaa397e4c..dc85788b6aef71 100644
--- a/libc/src/__support/freelist2.h
+++ b/libc/src/__support/freelist.h
@@ -6,22 +6,22 @@
//
//===----------------------------------------------------------------------===//
-#ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST2_H
-#define LLVM_LIBC_SRC___SUPPORT_FREELIST2_H
+#ifndef LLVM_LIBC_SRC___SUPPORT_FREELIST_H
+#define LLVM_LIBC_SRC___SUPPORT_FREELIST_H
#include "block.h"
namespace LIBC_NAMESPACE_DECL {
/// A circularly-linked FIFO list node storing a free Block. A list is a
-/// FreeList2*; nullptr is an empty list. All Blocks on a list are the same
+/// FreeList*; nullptr is an empty list. All Blocks on a list are the same
/// size.
///
/// Accessing free blocks in FIFO order maximizes the amount of time before a
/// free block is reused. This in turn maximizes the number of opportunities for
/// it to be coalesced with an adjacent block, which tends to reduce heap
/// fragmentation.
-class FreeList2 {
+class FreeList {
public:
Block<> *block() const {
return const_cast<Block<> *>(Block<>::from_usable_space(this));
@@ -30,30 +30,30 @@ class FreeList2 {
/// @returns Size for all blocks on the list.
size_t size() const { return block()->inner_size(); }
- /// Push to the back. The Block must be able to contain a FreeList2.
- static void push(FreeList2 *&list, Block<> *block);
+ /// Push to the back. The Block must be able to contain a FreeList.
+ static void push(FreeList *&list, Block<> *block);
/// Pop the front.
- static void pop(FreeList2 *&list);
+ static void pop(FreeList *&list);
protected:
/// Push an already-constructed node to the back.
- static void push(FreeList2 *&list, FreeList2 *node);
+ static void push(FreeList *&list, FreeList *node);
private:
// Circularly linked pointers to adjacent nodes.
- FreeList2 *prev;
- FreeList2 *next;
+ FreeList *prev;
+ FreeList *next;
};
-LIBC_INLINE void FreeList2::push(FreeList2 *&list, Block<> *block) {
+LIBC_INLINE void FreeList::push(FreeList *&list, Block<> *block) {
LIBC_ASSERT(!block->used() && "only free blocks can be placed on free lists");
- LIBC_ASSERT(block->inner_size_free() >= sizeof(FreeList2) &&
+ LIBC_ASSERT(block->inner_size_free() >= sizeof(FreeList) &&
"block too small to accomodate free list node");
- push(list, new (block->usable_space()) FreeList2);
+ push(list, new (block->usable_space()) FreeList);
}
-LIBC_INLINE void FreeList2::pop(FreeList2 *&list) {
+LIBC_INLINE void FreeList::pop(FreeList *&list) {
LIBC_ASSERT(list != nullptr && "cannot pop from empty list");
if (list->next == list) {
list = nullptr;
@@ -64,7 +64,7 @@ LIBC_INLINE void FreeList2::pop(FreeList2 *&list) {
}
}
-LIBC_INLINE void FreeList2::push(FreeList2 *&list, FreeList2 *node) {
+LIBC_INLINE void FreeList::push(FreeList *&list, FreeList *node) {
if (list) {
LIBC_ASSERT(Block<>::from_usable_space(node)->outer_size() ==
list->block()->outer_size() &&
@@ -80,4 +80,4 @@ LIBC_INLINE void FreeList2::push(FreeList2 *&list, FreeList2 *node) {
} // namespace LIBC_NAMESPACE_DECL
-#endif // LLVM_LIBC_SRC___SUPPORT_FREELIST2_H
+#endif // LLVM_LIBC_SRC___SUPPORT_FREELIST_H
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 1d71055a521dd8..309c629145f93b 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -26,7 +26,7 @@ class FreeStore {
Block<> *remove_best_fit(size_t size);
private:
- static constexpr size_t MIN_SIZE = sizeof(FreeList2);
+ static constexpr size_t MIN_SIZE = sizeof(FreeList);
static constexpr size_t MIN_LARGE_SIZE = sizeof(FreeTrie);
static constexpr size_t NUM_SMALL_SIZES =
(align_up(MIN_LARGE_SIZE) - align_up(MIN_SIZE)) / alignof(max_align_t);
@@ -34,10 +34,10 @@ class FreeStore {
static bool is_small(Block<> *block);
static bool is_small(size_t size);
- FreeList2 *&small_list(Block<> *block);
- FreeList2 **best_small_fit(size_t size);
+ FreeList *&small_list(Block<> *block);
+ FreeList **best_small_fit(size_t size);
- cpp::array<FreeList2 *, NUM_SMALL_SIZES> small_lists = {nullptr};
+ cpp::array<FreeList *, NUM_SMALL_SIZES> small_lists = {nullptr};
FreeTrie *large_trie = nullptr;
FreeTrie::SizeRange range = {0, 0};
};
@@ -51,7 +51,7 @@ inline void FreeStore::insert(Block<> *block) {
if (block->inner_size_free() < MIN_SIZE)
return;
if (is_small(block))
- FreeList2::push(small_list(block), block);
+ FreeList::push(small_list(block), block);
else
FreeTrie::push(FreeTrie::find(large_trie, block->inner_size(), range),
block);
@@ -61,13 +61,13 @@ inline void FreeStore::remove(Block<> *block) {
LIBC_ASSERT(block->inner_size_free() >= MIN_SIZE &&
"block too small to have been present");
if (is_small(block)) {
- FreeList2 *block_list =
- reinterpret_cast<FreeList2 *>(block->usable_space());
- FreeList2 *&list = small_list(block);
+ FreeList *block_list =
+ reinterpret_cast<FreeList *>(block->usable_space());
+ FreeList *&list = small_list(block);
if (block_list == list)
- FreeList2::pop(list);
+ FreeList::pop(list);
else
- FreeList2::pop(block_list);
+ FreeList::pop(block_list);
} else {
auto *trie = reinterpret_cast<FreeTrie *>(block->usable_space());
if (trie == large_trie) {
@@ -80,11 +80,11 @@ inline void FreeStore::remove(Block<> *block) {
inline Block<> *FreeStore::remove_best_fit(size_t size) {
if (is_small(size)) {
- FreeList2 **list = best_small_fit(size);
+ FreeList **list = best_small_fit(size);
if (!list)
return nullptr;
Block<> *block = (*list)->block();
- FreeList2::pop(*list);
+ FreeList::pop(*list);
return block;
}
@@ -106,14 +106,14 @@ inline bool FreeStore::is_small(size_t size) {
return size - sizeof(Block<>::offset_type) < MIN_LARGE_SIZE;
}
-inline FreeList2 *&FreeStore::small_list(Block<> *block) {
+inline FreeList *&FreeStore::small_list(Block<> *block) {
LIBC_ASSERT(is_small(block) && "only legal for small blocks");
return small_lists[(block->inner_size_free() - MIN_SIZE) /
alignof(max_align_t)];
}
-inline FreeList2 **FreeStore::best_small_fit(size_t size) {
- for (FreeList2 *&list : small_lists)
+inline FreeList **FreeStore::best_small_fit(size_t size) {
+ for (FreeList *&list : small_lists)
if (list && list->size() >= size)
return &list;
return nullptr;
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 8e0fb0f2d7495c..59df0c444fc9fb 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -10,14 +10,14 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
#define LLVM_LIBC_SRC___SUPPORT_FREETRIE_H
-#include "freelist2.h"
+#include "freelist.h"
namespace LIBC_NAMESPACE_DECL {
/// A trie node containing a free list. The subtrie contains a contiguous
/// SizeRange of freelists.There is no relationship between the size of this
/// free list and the size ranges of the subtries.
-class FreeTrie : public FreeList2 {
+class FreeTrie : public FreeList {
public:
// Power-of-two range of sizes covered by a subtrie.
struct SizeRange {
@@ -112,14 +112,14 @@ LIBC_INLINE void FreeTrie::push(InsertPos pos, Block<> *block) {
node->lower = node->upper = nullptr;
node->parent = pos.parent;
}
- FreeList2 *list = *pos.trie;
- FreeList2::push(list, node);
+ FreeList *list = *pos.trie;
+ FreeList::push(list, node);
*pos.trie = static_cast<FreeTrie *>(list);
}
LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
- FreeList2 *list = trie;
- FreeList2::pop(list);
+ FreeList *list = trie;
+ FreeList::pop(list);
FreeTrie *new_trie = static_cast<FreeTrie *>(list);
if (new_trie) {
// The freelist is non-empty, so copy the trie links to the new head.
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 1bda590b55e102..4670a022609b18 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -20,7 +20,7 @@ if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
SUITE
libc-support-tests
SRCS
- freelist2_test.cpp
+ freelist_test.cpp
#freetrie_test.cpp
DEPENDS
libc.src.__support.CPP.array
diff --git a/libc/test/src/__support/freelist2_test.cpp b/libc/test/src/__support/freelist_test.cpp
similarity index 70%
rename from libc/test/src/__support/freelist2_test.cpp
rename to libc/test/src/__support/freelist_test.cpp
index a6418883d65503..68e96cb555032f 100644
--- a/libc/test/src/__support/freelist2_test.cpp
+++ b/libc/test/src/__support/freelist_test.cpp
@@ -8,12 +8,12 @@
#include <stddef.h>
-#include "src/__support/freelist2.h"
+#include "src/__support/freelist.h"
#include "test/UnitTest/Test.h"
namespace LIBC_NAMESPACE_DECL {
-TEST(LlvmLibcFreeList2, PushPop) {
+TEST(LlvmLibcFreeList, PushPop) {
cpp::byte mem1[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem1);
ASSERT_TRUE(maybeBlock.has_value());
@@ -24,20 +24,20 @@ TEST(LlvmLibcFreeList2, PushPop) {
ASSERT_TRUE(maybeBlock.has_value());
Block<> *block2 = *maybeBlock;
- FreeList2 *list = nullptr;
- FreeList2::push(list, block1);
- ASSERT_NE(list, static_cast<FreeList2*>(nullptr));
+ FreeList *list = nullptr;
+ FreeList::push(list, block1);
+ ASSERT_NE(list, static_cast<FreeList *>(nullptr));
EXPECT_EQ(list->block(), block1);
- FreeList2::push(list, block2);
+ FreeList::push(list, block2);
EXPECT_EQ(list->block(), block1);
- FreeList2::pop(list);
- ASSERT_NE(list, static_cast<FreeList2*>(nullptr));
+ FreeList::pop(list);
+ ASSERT_NE(list, static_cast<FreeList *>(nullptr));
EXPECT_EQ(list->block(), block2);
- FreeList2::pop(list);
- ASSERT_EQ(list, static_cast<FreeList2*>(nullptr));
+ FreeList::pop(list);
+ ASSERT_EQ(list, static_cast<FreeList *>(nullptr));
}
} // namespace LIBC_NAMESPACE_DECL
>From cf876b91540098ff2008a30e0654b7162de541fc Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 27 Aug 2024 11:07:01 -0700
Subject: [PATCH 44/54] Simplify freetrie interface for how it's used
---
libc/src/__support/freestore.h | 15 +++-----
libc/src/__support/freetrie.h | 69 +++++++++++++---------------------
2 files changed, 33 insertions(+), 51 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 309c629145f93b..8039f19b8d350c 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -53,16 +53,14 @@ inline void FreeStore::insert(Block<> *block) {
if (is_small(block))
FreeList::push(small_list(block), block);
else
- FreeTrie::push(FreeTrie::find(large_trie, block->inner_size(), range),
- block);
+ FreeTrie::push(large_trie, block, range);
}
inline void FreeStore::remove(Block<> *block) {
LIBC_ASSERT(block->inner_size_free() >= MIN_SIZE &&
"block too small to have been present");
if (is_small(block)) {
- FreeList *block_list =
- reinterpret_cast<FreeList *>(block->usable_space());
+ FreeList *block_list = reinterpret_cast<FreeList *>(block->usable_space());
FreeList *&list = small_list(block);
if (block_list == list)
FreeList::pop(list);
@@ -70,11 +68,10 @@ inline void FreeStore::remove(Block<> *block) {
FreeList::pop(block_list);
} else {
auto *trie = reinterpret_cast<FreeTrie *>(block->usable_space());
- if (trie == large_trie) {
- FreeTrie::pop(large_trie);
- } else {
- FreeTrie::pop(trie->self());
- }
+ if (trie == large_trie)
+ FreeTrie::pop(large_trie); // Update large_trie if necessary.
+ else
+ FreeTrie::pop(trie);
}
}
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 59df0c444fc9fb..28c04f36fa379c 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -40,24 +40,13 @@ class FreeTrie : public FreeList {
bool contains(size_t size) const;
};
- struct InsertPos {
- FreeTrie *parent;
- FreeTrie **trie;
- };
-
- FreeTrie *&self();
-
- /// Push to the back of this node's free list.
- static void push(InsertPos pos, Block<> *block);
+ /// Push to the the correct free list in this trie. The caller must provide
+ /// the SizeRange for this trie; the trie does not store it.
+ static void push(FreeTrie *&trie, Block<> *block, SizeRange range);
/// Pop from the front of this node's free list.
static void pop(FreeTrie *&trie);
- /// Finds the free trie for a given size. This may be a referance to a nullptr
- /// at the correct place in the trie structure. The caller must provide the
- /// SizeRange for this trie; the trie does not store it.
- static InsertPos find(FreeTrie *&trie, size_t size, SizeRange range);
-
static FreeTrie **find_best_fit(FreeTrie *&trie, size_t size,
SizeRange range);
@@ -73,11 +62,6 @@ class FreeTrie : public FreeList {
FreeTrie *parent;
};
-inline FreeTrie *&FreeTrie::self() {
- LIBC_ASSERT(parent && "reference in parent not defined on root");
- return parent->lower == this ? parent->lower : parent->upper;
-}
-
LIBC_INLINE constexpr FreeTrie::SizeRange::SizeRange(size_t min, size_t width)
: min(min), width(width) {
LIBC_ASSERT(!(width & (width - 1)) && "width must be a power of two");
@@ -103,18 +87,37 @@ LIBC_INLINE bool FreeTrie::SizeRange::contains(size_t size) const {
return true;
}
-LIBC_INLINE void FreeTrie::push(InsertPos pos, Block<> *block) {
+LIBC_INLINE void FreeTrie::push(FreeTrie *&trie, Block<> *block,
+ SizeRange range) {
LIBC_ASSERT(block->inner_size_free() >= sizeof(FreeTrie) &&
"block too small to accomodate free trie node");
+
+ size_t size = block->inner_size();
+ LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
+
+ FreeTrie **cur = ≜
+ FreeTrie *parent = nullptr;
+ while (*cur && (*cur)->size() != size) {
+ LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
+ parent = *cur;
+ if (range.lower().contains(size)) {
+ cur = &(*cur)->lower;
+ range = range.lower();
+ } else {
+ cur = &(*cur)->upper;
+ range = range.upper();
+ }
+ }
+
FreeTrie *node = new (block->usable_space()) FreeTrie;
// The trie links are irrelevant for all but the first node in the free list.
- if (!*pos.trie) {
+ if (!*cur) {
node->lower = node->upper = nullptr;
- node->parent = pos.parent;
+ node->parent = parent;
}
- FreeList *list = *pos.trie;
+ FreeList *list = *cur;
FreeList::push(list, node);
- *pos.trie = static_cast<FreeTrie *>(list);
+ *cur = static_cast<FreeTrie *>(list);
}
LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
@@ -147,24 +150,6 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
trie = &l;
}
-LIBC_INLINE FreeTrie::InsertPos FreeTrie::find(FreeTrie *&trie, size_t size,
- SizeRange range) {
- LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
- InsertPos pos = {nullptr, &trie};
- while (*pos.trie && (*pos.trie)->size() != size) {
- LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
- pos.parent = *pos.trie;
- if (range.lower().contains(size)) {
- pos.trie = &(*pos.trie)->lower;
- range = range.lower();
- } else {
- pos.trie = &(*pos.trie)->upper;
- range = range.upper();
- }
- }
- return pos;
-}
-
LIBC_INLINE FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
SizeRange range) {
if (!trie)
>From db7546103863a50150d278b15a9eb7298d754d60 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 27 Aug 2024 13:31:30 -0700
Subject: [PATCH 45/54] Rewrite freetrie test
---
libc/test/src/__support/CMakeLists.txt | 2 +-
libc/test/src/__support/freetrie_test.cpp | 21 +++++++++++++--------
2 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 4670a022609b18..a1cae3b4fee77f 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -21,7 +21,7 @@ if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
libc-support-tests
SRCS
freelist_test.cpp
- #freetrie_test.cpp
+ freetrie_test.cpp
DEPENDS
libc.src.__support.CPP.array
libc.src.__support.CPP.span
diff --git a/libc/test/src/__support/freetrie_test.cpp b/libc/test/src/__support/freetrie_test.cpp
index a3491f2ac382a5..f9714266693410 100644
--- a/libc/test/src/__support/freetrie_test.cpp
+++ b/libc/test/src/__support/freetrie_test.cpp
@@ -23,7 +23,7 @@ template <size_t size> struct BlockMem {
Block<> *block;
};
-TEST(LlvmLibcFreeTrie, PushPop) {
+TEST(LlvmLibcFreeTrie, PushPopRoot) {
BlockMem<1024> block1_mem;
Block<> *block1 = block1_mem.block;
BlockMem<1024> block2_mem;
@@ -31,15 +31,17 @@ TEST(LlvmLibcFreeTrie, PushPop) {
BlockMem<1024> block3_mem;
Block<> *block3 = block3_mem.block;
+ FreeTrie::SizeRange range = {0, 4096};
+
FreeTrie *trie = nullptr;
- FreeTrie::push(trie, block1);
+ FreeTrie::push(trie, block1, range);
ASSERT_NE(trie, static_cast<FreeTrie *>(nullptr));
EXPECT_EQ(trie->block(), block1);
// Pushing blocks doesn't change the next block.
- FreeTrie::push(trie, block2);
+ FreeTrie::push(trie, block2, range);
EXPECT_EQ(trie->block(), block1);
- FreeTrie::push(trie, block3);
+ FreeTrie::push(trie, block3, range);
EXPECT_EQ(trie->block(), block1);
// Blocks are popped in FIFO order
@@ -55,12 +57,12 @@ TEST(LlvmLibcFreeTrie, PushPop) {
ASSERT_EQ(trie, static_cast<FreeTrie *>(nullptr));
}
-TEST(LlvmLibcFreeTrie, Find) {
- // Finding in an empty trie returns the trie itself.
+TEST(LlvmLibcFreeTrie, PushFind) {
FreeTrie *trie = nullptr;
- FreeTrie *&empty_found = FreeTrie::find(trie, 123, {0, 1024});
- EXPECT_EQ(&empty_found, &trie);
+ EXPECT_EQ(FreeTrie::find_best_fit(trie, 123, {0, 1024}),
+ static_cast<FreeTrie **>(nullptr));
+#if 0
BlockMem<768> block_mem;
Block<> *block = block_mem.block;
FreeTrie::push(trie, block);
@@ -85,8 +87,10 @@ TEST(LlvmLibcFreeTrie, Find) {
FreeTrie *&upper2_found = FreeTrie::find(trie, 1024 - 1, {0, 1024});
EXPECT_EQ(&upper2_found, &upper_found);
+#endif
}
+#if 0
TEST(LlvmLibcFreeTrie, PopPreservesChildren) {
FreeTrie::SizeRange range{0, 4096};
@@ -199,5 +203,6 @@ TEST(LlvmLibcFreeTrie, FindBestFitLowerAndUpper) {
// subtrie.
EXPECT_EQ(FreeTrie::find_best_fit(trie, range.width / 2, range), &upper);
}
+#endif
} // namespace LIBC_NAMESPACE_DECL
>From 9fbda1011ad2ba5d445907a2d1d95ee382253132 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 27 Aug 2024 14:13:57 -0700
Subject: [PATCH 46/54] Make naming clearer
---
libc/src/__support/freestore.h | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 8039f19b8d350c..46be5ce1f46ce3 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -60,16 +60,16 @@ inline void FreeStore::remove(Block<> *block) {
LIBC_ASSERT(block->inner_size_free() >= MIN_SIZE &&
"block too small to have been present");
if (is_small(block)) {
- FreeList *block_list = reinterpret_cast<FreeList *>(block->usable_space());
- FreeList *&list = small_list(block);
- if (block_list == list)
- FreeList::pop(list);
+ FreeList *list = reinterpret_cast<FreeList *>(block->usable_space());
+ FreeList *&list_head = small_list(block);
+ if (list == list_head)
+ FreeList::pop(list_head);
else
- FreeList::pop(block_list);
+ FreeList::pop(list);
} else {
auto *trie = reinterpret_cast<FreeTrie *>(block->usable_space());
if (trie == large_trie)
- FreeTrie::pop(large_trie); // Update large_trie if necessary.
+ FreeTrie::pop(large_trie);
else
FreeTrie::pop(trie);
}
>From d2ac5b0e98d2d21f0d004c3fdbd010dfd3b7989f Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 27 Aug 2024 14:24:32 -0700
Subject: [PATCH 47/54] Update parent link when child replaced by pop
---
libc/src/__support/freetrie.h | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 28c04f36fa379c..de95f9aa7f90f4 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -51,6 +51,8 @@ class FreeTrie : public FreeList {
SizeRange range);
private:
+ static void replace_trie(FreeTrie *&trie, FreeTrie *new_trie);
+
/// Return an abitrary leaf.
FreeTrie &leaf();
@@ -121,6 +123,7 @@ LIBC_INLINE void FreeTrie::push(FreeTrie *&trie, Block<> *block,
}
LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
+ LIBC_ASSERT(trie && "cannot pop from empty trie");
FreeList *list = trie;
FreeList::pop(list);
FreeTrie *new_trie = static_cast<FreeTrie *>(list);
@@ -129,7 +132,7 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
new_trie->lower = trie->lower;
new_trie->upper = trie->upper;
new_trie->parent = trie->parent;
- trie = new_trie;
+ replace_trie(trie, new_trie);
return;
}
@@ -138,7 +141,7 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
FreeTrie &l = trie->leaf();
if (&l == trie) {
// If the root is a leaf, then removing it empties the trie.
- trie = nullptr;
+ replace_trie(trie, nullptr);
return;
}
@@ -147,7 +150,7 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
l.lower = trie->lower;
l.upper = trie->upper;
l.parent = trie->parent;
- trie = &l;
+ replace_trie(trie, &l);
}
LIBC_INLINE FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
@@ -222,6 +225,15 @@ LIBC_INLINE FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
}
}
+LIBC_INLINE void FreeTrie::replace_trie(FreeTrie *&trie, FreeTrie *new_trie) {
+ if (trie && trie->parent) {
+ FreeTrie *&parent_child =
+ trie->parent->lower == trie ? trie->parent->lower : trie->parent->upper;
+ parent_child = new_trie;
+ }
+ trie = new_trie;
+}
+
LIBC_INLINE FreeTrie &FreeTrie::leaf() {
FreeTrie *cur = this;
while (cur->lower || cur->upper)
>From 36d22a5168fb4327fb98cad1fc203143f1cf4266 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 27 Aug 2024 15:45:15 -0700
Subject: [PATCH 48/54] Allow out of range find
---
libc/src/__support/freetrie.h | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index de95f9aa7f90f4..3407c8f1d5091f 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -155,10 +155,9 @@ LIBC_INLINE void FreeTrie::pop(FreeTrie *&trie) {
LIBC_INLINE FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
SizeRange range) {
- if (!trie)
+ if (!trie || range.max() < size)
return nullptr;
- LIBC_ASSERT(range.contains(size) && "requested size out of trie range");
FreeTrie **cur = ≜
FreeTrie **best_fit = nullptr;
FreeTrie **deferred_upper_trie = nullptr;
>From d8cc3ed90a6b0910e8972f772f504c6b68c3d9fa Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 27 Aug 2024 15:50:02 -0700
Subject: [PATCH 49/54] Allow large fits for small requests
---
libc/src/__support/freestore.h | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 46be5ce1f46ce3..3e3730dc83c2fd 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -42,12 +42,12 @@ class FreeStore {
FreeTrie::SizeRange range = {0, 0};
};
-inline void FreeStore::set_range(FreeTrie::SizeRange range) {
+LIBC_INLINE void FreeStore::set_range(FreeTrie::SizeRange range) {
LIBC_ASSERT(!large_trie && "cannot change the range of a preexisting trie");
this->range = range;
}
-inline void FreeStore::insert(Block<> *block) {
+LIBC_INLINE void FreeStore::insert(Block<> *block) {
if (block->inner_size_free() < MIN_SIZE)
return;
if (is_small(block))
@@ -56,7 +56,7 @@ inline void FreeStore::insert(Block<> *block) {
FreeTrie::push(large_trie, block, range);
}
-inline void FreeStore::remove(Block<> *block) {
+LIBC_INLINE void FreeStore::remove(Block<> *block) {
LIBC_ASSERT(block->inner_size_free() >= MIN_SIZE &&
"block too small to have been present");
if (is_small(block)) {
@@ -75,14 +75,14 @@ inline void FreeStore::remove(Block<> *block) {
}
}
-inline Block<> *FreeStore::remove_best_fit(size_t size) {
+LIBC_INLINE Block<> *FreeStore::remove_best_fit(size_t size) {
if (is_small(size)) {
FreeList **list = best_small_fit(size);
- if (!list)
- return nullptr;
- Block<> *block = (*list)->block();
- FreeList::pop(*list);
- return block;
+ if (list) {
+ Block<> *block = (*list)->block();
+ FreeList::pop(*list);
+ return block;
+ }
}
FreeTrie **best_fit = FreeTrie::find_best_fit(large_trie, size, range);
@@ -93,23 +93,23 @@ inline Block<> *FreeStore::remove_best_fit(size_t size) {
return block;
}
-inline bool FreeStore::is_small(Block<> *block) {
+LIBC_INLINE bool FreeStore::is_small(Block<> *block) {
return block->inner_size_free() < MIN_LARGE_SIZE;
}
-inline bool FreeStore::is_small(size_t size) {
+LIBC_INLINE bool FreeStore::is_small(size_t size) {
if (size < sizeof(Block<>::offset_type))
return true;
return size - sizeof(Block<>::offset_type) < MIN_LARGE_SIZE;
}
-inline FreeList *&FreeStore::small_list(Block<> *block) {
+LIBC_INLINE FreeList *&FreeStore::small_list(Block<> *block) {
LIBC_ASSERT(is_small(block) && "only legal for small blocks");
return small_lists[(block->inner_size_free() - MIN_SIZE) /
alignof(max_align_t)];
}
-inline FreeList **FreeStore::best_small_fit(size_t size) {
+LIBC_INLINE FreeList **FreeStore::best_small_fit(size_t size) {
for (FreeList *&list : small_lists)
if (list && list->size() >= size)
return &list;
>From cc363ef44a9c6328fbb7e6994300b453504449e3 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Wed, 28 Aug 2024 15:17:15 -0700
Subject: [PATCH 50/54] Reword comments
---
libc/src/__support/freelist.h | 6 +++---
libc/src/__support/freetrie.h | 8 +++++---
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/libc/src/__support/freelist.h b/libc/src/__support/freelist.h
index dc85788b6aef71..5cb33c95fa66b9 100644
--- a/libc/src/__support/freelist.h
+++ b/libc/src/__support/freelist.h
@@ -13,9 +13,9 @@
namespace LIBC_NAMESPACE_DECL {
-/// A circularly-linked FIFO list node storing a free Block. A list is a
-/// FreeList*; nullptr is an empty list. All Blocks on a list are the same
-/// size.
+/// A circularly-linked FIFO list node storing a free Block. A FreeList pointer
+/// represents a list; nullptr represents an empty list. All Blocks on a list
+/// are the same size.
///
/// Accessing free blocks in FIFO order maximizes the amount of time before a
/// free block is reused. This in turn maximizes the number of opportunities for
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index 3407c8f1d5091f..e1cba598c58135 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -14,9 +14,11 @@
namespace LIBC_NAMESPACE_DECL {
-/// A trie node containing a free list. The subtrie contains a contiguous
-/// SizeRange of freelists.There is no relationship between the size of this
-/// free list and the size ranges of the subtries.
+/// A trie node also acting as a free list. The subtrie collectively contains a
+/// contiguous SizeRange of free lists. The lower and upper subtrie's contain
+/// the lower and upper half of the subtries range. There is no direct
+/// relationship between the size of this free list and the contents of the
+/// lower and upper subtries.
class FreeTrie : public FreeList {
public:
// Power-of-two range of sizes covered by a subtrie.
>From 050b873e14a3a0663bb343ebec1ff073cdaa4a50 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Fri, 30 Aug 2024 15:28:19 -0700
Subject: [PATCH 51/54] Too small test
---
libc/src/__support/freestore.h | 4 ++--
libc/test/src/__support/freestore_test.cpp | 22 ++++++++++++++++++++--
2 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h
index 3e3730dc83c2fd..12f72fdae9ec59 100644
--- a/libc/src/__support/freestore.h
+++ b/libc/src/__support/freestore.h
@@ -57,8 +57,8 @@ LIBC_INLINE void FreeStore::insert(Block<> *block) {
}
LIBC_INLINE void FreeStore::remove(Block<> *block) {
- LIBC_ASSERT(block->inner_size_free() >= MIN_SIZE &&
- "block too small to have been present");
+ if (block->inner_size_free() < MIN_SIZE)
+ return;
if (is_small(block)) {
FreeList *list = reinterpret_cast<FreeList *>(block->usable_space());
FreeList *&list_head = small_list(block);
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index b593792d12ed42..973c5d61abcd0c 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -13,8 +13,26 @@
namespace LIBC_NAMESPACE_DECL {
-TEST(LlvmLibcFreeStore, PushPop) {
- EXPECT_EQ(1, 2);
+// Inserting blocks too small to be tracked does nothing.
+TEST(LlvmLibcFreeStore, TooSmall) {
+ cpp::byte mem[1024];
+ optional<Block<> *> maybeBlock = Block<>::init(mem);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *too_small = *maybeBlock;
+ maybeBlock =
+ too_small->split(sizeof(Block<>::offset_type));
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *remainder = *maybeBlock;
+
+ FreeStore store;
+ store.set_range({0, 4096});
+ store.insert(too_small);
+ store.insert(remainder);
+
+ // Inserting a too small block does nothing.
+ EXPECT_EQ(store.remove_best_fit(too_small->inner_size()), remainder);
+ // Removing a too small block has no effect.
+ store.remove(too_small);
}
} // namespace LIBC_NAMESPACE_DECL
>From 10f2c544b00be651be82a1cf8a2a5dfbcc9ed717 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Fri, 30 Aug 2024 16:28:17 -0700
Subject: [PATCH 52/54] Add test for small blocks
---
libc/test/src/__support/freestore_test.cpp | 36 +++++++++++++++++++---
1 file changed, 31 insertions(+), 5 deletions(-)
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index 973c5d61abcd0c..54ed4b9643463a 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -13,14 +13,13 @@
namespace LIBC_NAMESPACE_DECL {
-// Inserting blocks too small to be tracked does nothing.
+// Inserting or removing blocks too small to be tracked does nothing.
TEST(LlvmLibcFreeStore, TooSmall) {
cpp::byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
Block<> *too_small = *maybeBlock;
- maybeBlock =
- too_small->split(sizeof(Block<>::offset_type));
+ maybeBlock = too_small->split(sizeof(Block<>::offset_type));
ASSERT_TRUE(maybeBlock.has_value());
Block<> *remainder = *maybeBlock;
@@ -29,10 +28,37 @@ TEST(LlvmLibcFreeStore, TooSmall) {
store.insert(too_small);
store.insert(remainder);
- // Inserting a too small block does nothing.
EXPECT_EQ(store.remove_best_fit(too_small->inner_size()), remainder);
- // Removing a too small block has no effect.
store.remove(too_small);
}
+TEST(LlvmLibcFreeStore, Small) {
+ cpp::byte mem[1024];
+ optional<Block<> *> maybeBlock = Block<>::init(mem);
+ ASSERT_TRUE(maybeBlock.has_value());
+ size_t smallest_size = sizeof(FreeList) + sizeof(Block<>::offset_type);
+ Block<> *smallest = *maybeBlock;
+ maybeBlock = smallest->split(smallest_size);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *less_small = *maybeBlock;
+ maybeBlock = less_small->split(smallest_size + 1);
+ ASSERT_TRUE(maybeBlock.has_value());
+ Block<> *remainder = *maybeBlock;
+ ASSERT_GT(less_small->inner_size(), smallest->inner_size());
+
+ FreeStore store;
+ store.set_range({0, 4096});
+ store.insert(smallest);
+ store.insert(less_small);
+ store.insert(remainder);
+
+ ASSERT_EQ(store.remove_best_fit(smallest->inner_size()), smallest);
+ store.insert(smallest);
+ ASSERT_EQ(store.remove_best_fit(less_small->inner_size()), less_small);
+ store.insert(less_small);
+ ASSERT_EQ(store.remove_best_fit(smallest->inner_size() + 1), less_small);
+ store.insert(less_small);
+ EXPECT_EQ(store.remove_best_fit(less_small->inner_size() + 1), remainder);
+}
+
} // namespace LIBC_NAMESPACE_DECL
>From e24105095be81c845426352dccd250913753d6a9 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Tue, 3 Sep 2024 11:28:12 -0700
Subject: [PATCH 53/54] Test full range of small sizes
---
libc/test/src/__support/freestore_test.cpp | 30 +++++++++++++++-------
1 file changed, 21 insertions(+), 9 deletions(-)
diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp
index 54ed4b9643463a..927a1e12b8e3d5 100644
--- a/libc/test/src/__support/freestore_test.cpp
+++ b/libc/test/src/__support/freestore_test.cpp
@@ -36,29 +36,41 @@ TEST(LlvmLibcFreeStore, Small) {
cpp::byte mem[1024];
optional<Block<> *> maybeBlock = Block<>::init(mem);
ASSERT_TRUE(maybeBlock.has_value());
+
size_t smallest_size = sizeof(FreeList) + sizeof(Block<>::offset_type);
Block<> *smallest = *maybeBlock;
maybeBlock = smallest->split(smallest_size);
ASSERT_TRUE(maybeBlock.has_value());
- Block<> *less_small = *maybeBlock;
- maybeBlock = less_small->split(smallest_size + 1);
+
+ size_t largest_small_size =
+ align_up(sizeof(FreeTrie)) + sizeof(Block<>::offset_type);
+ Block<> *largest_small = *maybeBlock;
+ maybeBlock = largest_small->split(largest_small_size);
ASSERT_TRUE(maybeBlock.has_value());
+ ASSERT_GT(largest_small->inner_size(), smallest->inner_size());
+
Block<> *remainder = *maybeBlock;
- ASSERT_GT(less_small->inner_size(), smallest->inner_size());
FreeStore store;
store.set_range({0, 4096});
store.insert(smallest);
- store.insert(less_small);
+ store.insert(largest_small);
store.insert(remainder);
+ // Find exact match for smallest.
ASSERT_EQ(store.remove_best_fit(smallest->inner_size()), smallest);
store.insert(smallest);
- ASSERT_EQ(store.remove_best_fit(less_small->inner_size()), less_small);
- store.insert(less_small);
- ASSERT_EQ(store.remove_best_fit(smallest->inner_size() + 1), less_small);
- store.insert(less_small);
- EXPECT_EQ(store.remove_best_fit(less_small->inner_size() + 1), remainder);
+
+ // Find exact match for largest.
+ ASSERT_EQ(store.remove_best_fit(largest_small->inner_size()), largest_small);
+ store.insert(largest_small);
+
+ // Search smallest for best fit.
+ ASSERT_EQ(store.remove_best_fit(smallest->inner_size() + 1), largest_small);
+ store.insert(largest_small);
+
+ // Continue search for best fit to large blocks.
+ EXPECT_EQ(store.remove_best_fit(largest_small->inner_size() + 1), remainder);
}
} // namespace LIBC_NAMESPACE_DECL
>From f4b376085424eb19969f38cc21aa0087a8733f5e Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn at google.com>
Date: Thu, 5 Sep 2024 13:23:15 -0700
Subject: [PATCH 54/54] Misc fixes
---
libc/src/__support/freetrie.h | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/libc/src/__support/freetrie.h b/libc/src/__support/freetrie.h
index e1cba598c58135..d1cad60f5f4744 100644
--- a/libc/src/__support/freetrie.h
+++ b/libc/src/__support/freetrie.h
@@ -168,19 +168,20 @@ LIBC_INLINE FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
// Inductively assume all better fits than the current best are in the
// current subtrie.
while (true) {
+ size_t cur_size = (*cur)->size();
+ LIBC_ASSERT(range.contains(cur_size) && "trie node size out of range");
LIBC_ASSERT(range.max() >= size && "range could not fit requested size");
// If the current node is an exact fit, it is a best fit.
- if ((*cur)->size() == size)
+ if (cur_size == size)
return cur;
- if ((*cur)->size() > size &&
- (!best_fit || (*cur)->size() < (*best_fit)->size())) {
+ if (cur_size > size && (!best_fit || cur_size < (*best_fit)->size())) {
// The current node is a better fit.
best_fit = cur;
LIBC_ASSERT(
!deferred_upper_trie ||
- deferred_upper_range.min > (*cur)->size() &&
+ deferred_upper_range.min > cur_size &&
"deferred upper subtrie should be outclassed by new best fit");
deferred_upper_trie = nullptr;
}
@@ -216,12 +217,13 @@ LIBC_INLINE FreeTrie **FreeTrie::find_best_fit(FreeTrie *&trie, size_t size,
// Both subtries might contain a better fit. Any fit in the lower subtrie
// is better than the any fit in the upper subtrie, so scan the lower
// subtrie and return to the upper one if necessary.
- cur = &(*cur)->lower;
- range = range.lower();
LIBC_ASSERT(!deferred_upper_trie ||
range.upper().max() < deferred_upper_range.min &&
"old deferred upper subtrie should be outclassed by new");
deferred_upper_trie = &(*cur)->upper;
+ deferred_upper_range = range.upper();
+ cur = &(*cur)->lower;
+ range = range.lower();
}
}
}
More information about the libc-commits
mailing list