[llvm] [ADT] Add TrieRawHashMap (PR #69528)
Steven Wu via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 19 15:47:04 PDT 2023
https://github.com/cachemeifyoucan updated https://github.com/llvm/llvm-project/pull/69528
>From 6bc0b67cdd54408e4c9ba61cb574453a822db186 Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu at apple.com>
Date: Thu, 5 Oct 2023 13:02:40 -0700
Subject: [PATCH] [ADT] Add TrieRawHashMap
Implement TrieRawHashMap which stores objects into a Trie based on the
hash of the object.
User needs to supply the hashing function and guarantees the uniqueness of
the hash for the objects to be inserted. Hash collision is not
supported
---
llvm/include/llvm/ADT/TrieRawHashMap.h | 379 +++++++++++++++++
llvm/lib/Support/CMakeLists.txt | 1 +
llvm/lib/Support/TrieHashIndexGenerator.h | 88 ++++
llvm/lib/Support/TrieRawHashMap.cpp | 483 ++++++++++++++++++++++
llvm/unittests/ADT/CMakeLists.txt | 1 +
llvm/unittests/ADT/TrieRawHashMapTest.cpp | 346 ++++++++++++++++
6 files changed, 1298 insertions(+)
create mode 100644 llvm/include/llvm/ADT/TrieRawHashMap.h
create mode 100644 llvm/lib/Support/TrieHashIndexGenerator.h
create mode 100644 llvm/lib/Support/TrieRawHashMap.cpp
create mode 100644 llvm/unittests/ADT/TrieRawHashMapTest.cpp
diff --git a/llvm/include/llvm/ADT/TrieRawHashMap.h b/llvm/include/llvm/ADT/TrieRawHashMap.h
new file mode 100644
index 000000000000000..b7c4b92d1df307d
--- /dev/null
+++ b/llvm/include/llvm/ADT/TrieRawHashMap.h
@@ -0,0 +1,379 @@
+//===- TrieRawHashMap.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_TRIERAWHASHMAP_H
+#define LLVM_ADT_TRIERAWHASHMAP_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Casting.h"
+#include <atomic>
+#include <optional>
+
+namespace llvm {
+
+class raw_ostream;
+
+/// TrieRawHashMap - is a lock-free thread-safe trie that is can be used to
+/// store/index data based on a hash value. It can be customized to work with
+/// any hash algorithm or store any data.
+///
+/// Data structure:
+/// Data node stored in the Trie contains both hash and data:
+/// struct {
+/// HashT Hash;
+/// DataT Data;
+/// };
+///
+/// Data is stored/indexed via a prefix tree, where each node in the tree can be
+/// either the root, a sub-trie or a data node. Assuming a 4-bit hash and two
+/// data objects {0001, A} and {0100, B}, it can be stored in a trie
+/// (assuming Root has 2 bits, SubTrie has 1 bit):
+/// +--------+
+/// |Root[00]| -> {0001, A}
+/// | [01]| -> {0100, B}
+/// | [10]| (empty)
+/// | [11]| (empty)
+/// +--------+
+///
+/// Inserting a new object {0010, C} will result in:
+/// +--------+ +----------+
+/// |Root[00]| -> |SubTrie[0]| -> {0001, A}
+/// | | | [1]| -> {0010, C}
+/// | | +----------+
+/// | [01]| -> {0100, B}
+/// | [10]| (empty)
+/// | [11]| (empty)
+/// +--------+
+/// Note object A is sunk down to a sub-trie during the insertion. All the
+/// nodes are inserted through compare-exchange to ensure thread-safe and
+/// lock-free.
+///
+/// To find an object in the trie, walk the tree with prefix of the hash until
+/// the data node is found. Then the hash is compared with the hash stored in
+/// the data node to see if the is the same object.
+///
+/// Hash collision is not allowed so it is recommended to use trie with a
+/// "strong" hashing algorithm. A well-distributed hash can also result in
+/// better performance and memory usage.
+///
+/// It currently does not support iteration and deletion.
+
+/// Base class for a lock-free thread-safe hash-mapped trie.
+class ThreadSafeTrieRawHashMapBase {
+public:
+ static constexpr size_t TrieContentBaseSize = 4;
+ static constexpr size_t DefaultNumRootBits = 6;
+ static constexpr size_t DefaultNumSubtrieBits = 4;
+
+private:
+ template <class T> struct AllocValueType {
+ char Base[TrieContentBaseSize];
+ std::aligned_union_t<sizeof(T), T> Content;
+ };
+
+protected:
+ template <class T>
+ static constexpr size_t DefaultContentAllocSize = sizeof(AllocValueType<T>);
+
+ template <class T>
+ static constexpr size_t DefaultContentAllocAlign = alignof(AllocValueType<T>);
+
+ template <class T>
+ static constexpr size_t DefaultContentOffset =
+ offsetof(AllocValueType<T>, Content);
+
+public:
+ void operator delete(void *Ptr) { ::free(Ptr); }
+
+ LLVM_DUMP_METHOD void dump() const;
+ void print(raw_ostream &OS) const;
+
+protected:
+ /// Result of a lookup. Suitable for an insertion hint. Maybe could be
+ /// expanded into an iterator of sorts, but likely not useful (visiting
+ /// everything in the trie should probably be done some way other than
+ /// through an iterator pattern).
+ class PointerBase {
+ protected:
+ void *get() const { return I == -2u ? P : nullptr; }
+
+ public:
+ PointerBase() noexcept = default;
+
+ private:
+ friend class ThreadSafeTrieRawHashMapBase;
+ explicit PointerBase(void *Content) : P(Content), I(-2u) {}
+ PointerBase(void *P, unsigned I, unsigned B) : P(P), I(I), B(B) {}
+
+ bool isHint() const { return I != -1u && I != -2u; }
+
+ void *P = nullptr;
+ unsigned I = -1u;
+ unsigned B = 0;
+ };
+
+ /// Find the stored content with hash.
+ PointerBase find(ArrayRef<uint8_t> Hash) const;
+
+ /// Insert and return the stored content.
+ PointerBase
+ insert(PointerBase Hint, ArrayRef<uint8_t> Hash,
+ function_ref<const uint8_t *(void *Mem, ArrayRef<uint8_t> Hash)>
+ Constructor);
+
+ ThreadSafeTrieRawHashMapBase() = delete;
+
+ ThreadSafeTrieRawHashMapBase(
+ size_t ContentAllocSize, size_t ContentAllocAlign, size_t ContentOffset,
+ std::optional<size_t> NumRootBits = std::nullopt,
+ std::optional<size_t> NumSubtrieBits = std::nullopt);
+
+ /// Destructor, which asserts if there's anything to do. Subclasses should
+ /// call \a destroyImpl().
+ ///
+ /// \pre \a destroyImpl() was already called.
+ ~ThreadSafeTrieRawHashMapBase();
+ void destroyImpl(function_ref<void(void *ValueMem)> Destructor);
+
+ ThreadSafeTrieRawHashMapBase(ThreadSafeTrieRawHashMapBase &&RHS);
+
+ // Move assignment can be implemented in a thread-safe way if NumRootBits and
+ // NumSubtrieBits are stored inside the Root.
+ ThreadSafeTrieRawHashMapBase &
+ operator=(ThreadSafeTrieRawHashMapBase &&RHS) = delete;
+
+ // No copy.
+ ThreadSafeTrieRawHashMapBase(const ThreadSafeTrieRawHashMapBase &) = delete;
+ ThreadSafeTrieRawHashMapBase &
+ operator=(const ThreadSafeTrieRawHashMapBase &) = delete;
+
+ // Debug functions. Implementation details and not guaranteed to be
+ // thread-safe.
+ PointerBase getRoot() const;
+ unsigned getStartBit(PointerBase P) const;
+ unsigned getNumBits(PointerBase P) const;
+ unsigned getNumSlotUsed(PointerBase P) const;
+ std::string getTriePrefixAsString(PointerBase P) const;
+ unsigned getNumTries() const;
+ // Visit next trie in the allocation chain.
+ PointerBase getNextTrie(PointerBase P) const;
+
+private:
+ friend class TrieRawHashMapTestHelper;
+ const unsigned short ContentAllocSize;
+ const unsigned short ContentAllocAlign;
+ const unsigned short ContentOffset;
+ unsigned short NumRootBits;
+ unsigned short NumSubtrieBits;
+ struct ImplType;
+ // ImplPtr is owned by ThreadSafeTrieRawHashMapBase and needs to be freed in
+ // destoryImpl.
+ std::atomic<ImplType *> ImplPtr;
+ ImplType &getOrCreateImpl();
+ ImplType *getImpl() const;
+};
+
+/// Lock-free thread-safe hash-mapped trie.
+template <class T, size_t NumHashBytes>
+class ThreadSafeTrieRawHashMap : public ThreadSafeTrieRawHashMapBase {
+public:
+ using HashT = std::array<uint8_t, NumHashBytes>;
+
+ class LazyValueConstructor;
+ struct value_type {
+ const HashT Hash;
+ T Data;
+
+ value_type(value_type &&) = default;
+ value_type(const value_type &) = default;
+
+ value_type(ArrayRef<uint8_t> Hash, const T &Data)
+ : Hash(makeHash(Hash)), Data(Data) {}
+ value_type(ArrayRef<uint8_t> Hash, T &&Data)
+ : Hash(makeHash(Hash)), Data(std::move(Data)) {}
+
+ private:
+ friend class LazyValueConstructor;
+
+ struct EmplaceTag {};
+ template <class... ArgsT>
+ value_type(ArrayRef<uint8_t> Hash, EmplaceTag, ArgsT &&...Args)
+ : Hash(makeHash(Hash)), Data(std::forward<ArgsT>(Args)...) {}
+
+ static HashT makeHash(ArrayRef<uint8_t> HashRef) {
+ HashT Hash;
+ std::copy(HashRef.begin(), HashRef.end(), Hash.data());
+ return Hash;
+ }
+ };
+
+ using ThreadSafeTrieRawHashMapBase::operator delete;
+ using HashType = HashT;
+
+ using ThreadSafeTrieRawHashMapBase::dump;
+ using ThreadSafeTrieRawHashMapBase::print;
+
+private:
+ template <class ValueT> class PointerImpl : PointerBase {
+ friend class ThreadSafeTrieRawHashMap;
+
+ ValueT *get() const {
+ return reinterpret_cast<ValueT *>(PointerBase::get());
+ }
+
+ public:
+ ValueT &operator*() const {
+ assert(get());
+ return *get();
+ }
+ ValueT *operator->() const {
+ assert(get());
+ return get();
+ }
+ explicit operator bool() const { return get(); }
+
+ PointerImpl() = default;
+
+ protected:
+ PointerImpl(PointerBase Result) : PointerBase(Result) {}
+ };
+
+public:
+ class pointer;
+ class const_pointer;
+ class pointer : public PointerImpl<value_type> {
+ friend class ThreadSafeTrieRawHashMap;
+ friend class const_pointer;
+
+ public:
+ pointer() = default;
+
+ private:
+ pointer(PointerBase Result) : pointer::PointerImpl(Result) {}
+ };
+
+ class const_pointer : public PointerImpl<const value_type> {
+ friend class ThreadSafeTrieRawHashMap;
+
+ public:
+ const_pointer() = default;
+ const_pointer(const pointer &P) : const_pointer::PointerImpl(P) {}
+
+ private:
+ const_pointer(PointerBase Result) : const_pointer::PointerImpl(Result) {}
+ };
+
+ class LazyValueConstructor {
+ public:
+ value_type &operator()(T &&RHS) {
+ assert(Mem && "Constructor already called, or moved away");
+ return assign(::new (Mem) value_type(Hash, std::move(RHS)));
+ }
+ value_type &operator()(const T &RHS) {
+ assert(Mem && "Constructor already called, or moved away");
+ return assign(::new (Mem) value_type(Hash, RHS));
+ }
+ template <class... ArgsT> value_type &emplace(ArgsT &&...Args) {
+ assert(Mem && "Constructor already called, or moved away");
+ return assign(::new (Mem)
+ value_type(Hash, typename value_type::EmplaceTag{},
+ std::forward<ArgsT>(Args)...));
+ }
+
+ LazyValueConstructor(LazyValueConstructor &&RHS)
+ : Mem(RHS.Mem), Result(RHS.Result), Hash(RHS.Hash) {
+ RHS.Mem = nullptr; // Moved away, cannot call.
+ }
+ ~LazyValueConstructor() { assert(!Mem && "Constructor never called!"); }
+
+ private:
+ value_type &assign(value_type *V) {
+ Mem = nullptr;
+ Result = V;
+ return *V;
+ }
+ friend class ThreadSafeTrieRawHashMap;
+ LazyValueConstructor() = delete;
+ LazyValueConstructor(void *Mem, value_type *&Result, ArrayRef<uint8_t> Hash)
+ : Mem(Mem), Result(Result), Hash(Hash) {
+ assert(Hash.size() == sizeof(HashT) && "Invalid hash");
+ assert(Mem && "Invalid memory for construction");
+ }
+ void *Mem;
+ value_type *&Result;
+ ArrayRef<uint8_t> Hash;
+ };
+
+ /// Insert with a hint. Default-constructed hint will work, but it's
+ /// recommended to start with a lookup to avoid overhead in object creation
+ /// if it already exists.
+ pointer insertLazy(const_pointer Hint, ArrayRef<uint8_t> Hash,
+ function_ref<void(LazyValueConstructor)> OnConstruct) {
+ return pointer(ThreadSafeTrieRawHashMapBase::insert(
+ Hint, Hash, [&](void *Mem, ArrayRef<uint8_t> Hash) {
+ value_type *Result = nullptr;
+ OnConstruct(LazyValueConstructor(Mem, Result, Hash));
+ return Result->Hash.data();
+ }));
+ }
+
+ pointer insertLazy(ArrayRef<uint8_t> Hash,
+ function_ref<void(LazyValueConstructor)> OnConstruct) {
+ return insertLazy(const_pointer(), Hash, OnConstruct);
+ }
+
+ pointer insert(const_pointer Hint, value_type &&HashedData) {
+ return insertLazy(Hint, HashedData.Hash, [&](LazyValueConstructor C) {
+ C(std::move(HashedData.Data));
+ });
+ }
+
+ pointer insert(const_pointer Hint, const value_type &HashedData) {
+ return insertLazy(Hint, HashedData.Hash,
+ [&](LazyValueConstructor C) { C(HashedData.Data); });
+ }
+
+ pointer find(ArrayRef<uint8_t> Hash) {
+ assert(Hash.size() == std::tuple_size<HashT>::value);
+ return ThreadSafeTrieRawHashMapBase::find(Hash);
+ }
+
+ const_pointer find(ArrayRef<uint8_t> Hash) const {
+ assert(Hash.size() == std::tuple_size<HashT>::value);
+ return ThreadSafeTrieRawHashMapBase::find(Hash);
+ }
+
+ ThreadSafeTrieRawHashMap(std::optional<size_t> NumRootBits = std::nullopt,
+ std::optional<size_t> NumSubtrieBits = std::nullopt)
+ : ThreadSafeTrieRawHashMapBase(DefaultContentAllocSize<value_type>,
+ DefaultContentAllocAlign<value_type>,
+ DefaultContentOffset<value_type>,
+ NumRootBits, NumSubtrieBits) {}
+
+ ~ThreadSafeTrieRawHashMap() {
+ if constexpr (std::is_trivially_destructible<value_type>::value)
+ this->destroyImpl(nullptr);
+ else
+ this->destroyImpl(
+ [](void *P) { static_cast<value_type *>(P)->~value_type(); });
+ }
+
+ // Move constructor okay.
+ ThreadSafeTrieRawHashMap(ThreadSafeTrieRawHashMap &&) = default;
+
+ // No move assignment or any copy.
+ ThreadSafeTrieRawHashMap &operator=(ThreadSafeTrieRawHashMap &&) = delete;
+ ThreadSafeTrieRawHashMap(const ThreadSafeTrieRawHashMap &) = delete;
+ ThreadSafeTrieRawHashMap &
+ operator=(const ThreadSafeTrieRawHashMap &) = delete;
+};
+
+} // namespace llvm
+
+#endif // LLVM_ADT_TRIERAWHASHMAP_H
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index b96d62c7a6224d6..677f52677a27f8a 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -239,6 +239,7 @@ add_llvm_component_library(LLVMSupport
TimeProfiler.cpp
Timer.cpp
ToolOutputFile.cpp
+ TrieRawHashMap.cpp
Twine.cpp
TypeSize.cpp
Unicode.cpp
diff --git a/llvm/lib/Support/TrieHashIndexGenerator.h b/llvm/lib/Support/TrieHashIndexGenerator.h
new file mode 100644
index 000000000000000..fc1ebe92377a618
--- /dev/null
+++ b/llvm/lib/Support/TrieHashIndexGenerator.h
@@ -0,0 +1,88 @@
+//===- TrieHashIndexGenerator.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_SUPPORT_TRIEHASHINDEXGENERATOR_H
+#define LLVM_LIB_SUPPORT_TRIEHASHINDEXGENERATOR_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include <optional>
+
+namespace llvm {
+
+struct IndexGenerator {
+ size_t NumRootBits;
+ size_t NumSubtrieBits;
+ ArrayRef<uint8_t> Bytes;
+ std::optional<size_t> StartBit = std::nullopt;
+
+ size_t getNumBits() const {
+ assert(StartBit);
+ size_t TotalNumBits = Bytes.size() * 8;
+ assert(*StartBit <= TotalNumBits);
+ return std::min(*StartBit ? NumSubtrieBits : NumRootBits,
+ TotalNumBits - *StartBit);
+ }
+ size_t next() {
+ size_t Index;
+ if (!StartBit) {
+ StartBit = 0;
+ Index = getIndex(Bytes, *StartBit, NumRootBits);
+ } else {
+ *StartBit += *StartBit ? NumSubtrieBits : NumRootBits;
+ assert((*StartBit - NumRootBits) % NumSubtrieBits == 0);
+ Index = getIndex(Bytes, *StartBit, NumSubtrieBits);
+ }
+ return Index;
+ }
+
+ size_t hint(unsigned Index, unsigned Bit) {
+ assert(Bit < Bytes.size() * 8);
+ assert(Bit == 0 || (Bit - NumRootBits) % NumSubtrieBits == 0);
+ StartBit = Bit;
+ return Index;
+ }
+
+ size_t getCollidingBits(ArrayRef<uint8_t> CollidingBits) const {
+ assert(StartBit);
+ return getIndex(CollidingBits, *StartBit, NumSubtrieBits);
+ }
+
+ static size_t getIndex(ArrayRef<uint8_t> Bytes, size_t StartBit,
+ size_t NumBits) {
+ assert(StartBit < Bytes.size() * 8);
+
+ Bytes = Bytes.drop_front(StartBit / 8u);
+ StartBit %= 8u;
+ size_t Index = 0;
+ for (uint8_t Byte : Bytes) {
+ size_t ByteStart = 0, ByteEnd = 8;
+ if (StartBit) {
+ ByteStart = StartBit;
+ Byte &= (1u << (8 - StartBit)) - 1u;
+ StartBit = 0;
+ }
+ size_t CurrentNumBits = ByteEnd - ByteStart;
+ if (CurrentNumBits > NumBits) {
+ Byte >>= CurrentNumBits - NumBits;
+ CurrentNumBits = NumBits;
+ }
+ Index <<= CurrentNumBits;
+ Index |= Byte & ((1u << CurrentNumBits) - 1u);
+
+ assert(NumBits >= CurrentNumBits);
+ NumBits -= CurrentNumBits;
+ if (!NumBits)
+ break;
+ }
+ return Index;
+ }
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_SUPPORT_TRIEHASHINDEXGENERATOR_H
diff --git a/llvm/lib/Support/TrieRawHashMap.cpp b/llvm/lib/Support/TrieRawHashMap.cpp
new file mode 100644
index 000000000000000..3a4656d43621206
--- /dev/null
+++ b/llvm/lib/Support/TrieRawHashMap.cpp
@@ -0,0 +1,483 @@
+//===- TrieRawHashMap.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/TrieRawHashMap.h"
+#include "TrieHashIndexGenerator.h"
+#include "llvm/ADT/LazyAtomicPointer.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ThreadSafeAllocator.h"
+#include "llvm/Support/raw_ostream.h"
+#include <memory>
+
+using namespace llvm;
+
+namespace {
+struct TrieNode {
+ const bool IsSubtrie = false;
+
+ TrieNode(bool IsSubtrie) : IsSubtrie(IsSubtrie) {}
+
+ static void *operator new(size_t Size) { return ::malloc(Size); }
+ void operator delete(void *Ptr) { ::free(Ptr); }
+};
+
+struct TrieContent final : public TrieNode {
+ const uint8_t ContentOffset;
+ const uint8_t HashSize;
+ const uint8_t HashOffset;
+
+ void *getValuePointer() const {
+ auto *Content = reinterpret_cast<const uint8_t *>(this) + ContentOffset;
+ return const_cast<uint8_t *>(Content);
+ }
+
+ ArrayRef<uint8_t> getHash() const {
+ auto *Begin = reinterpret_cast<const uint8_t *>(this) + HashOffset;
+ return ArrayRef(Begin, Begin + HashSize);
+ }
+
+ TrieContent(size_t ContentOffset, size_t HashSize, size_t HashOffset)
+ : TrieNode(/*IsSubtrie=*/false), ContentOffset(ContentOffset),
+ HashSize(HashSize), HashOffset(HashOffset) {}
+
+ static bool classof(const TrieNode *TN) { return !TN->IsSubtrie; }
+};
+
+static_assert(sizeof(TrieContent) ==
+ ThreadSafeTrieRawHashMapBase::TrieContentBaseSize,
+ "Check header assumption!");
+
+class TrieSubtrie final : public TrieNode {
+public:
+ TrieNode *get(size_t I) const { return Slots[I].load(); }
+
+ TrieSubtrie *
+ sink(size_t I, TrieContent &Content, size_t NumSubtrieBits, size_t NewI,
+ function_ref<TrieSubtrie *(std::unique_ptr<TrieSubtrie>)> Saver);
+
+ static std::unique_ptr<TrieSubtrie> create(size_t StartBit, size_t NumBits);
+
+ explicit TrieSubtrie(size_t StartBit, size_t NumBits);
+
+ static bool classof(const TrieNode *TN) { return TN->IsSubtrie; }
+
+private:
+ // FIXME: Use a bitset to speed up access:
+ //
+ // std::array<std::atomic<uint64_t>, NumSlots/64> IsSet;
+ //
+ // This will avoid needing to visit sparsely filled slots in
+ // \a ThreadSafeTrieRawHashMapBase::destroyImpl() when there's a non-trivial
+ // destructor.
+ //
+ // It would also greatly speed up iteration, if we add that some day, and
+ // allow get() to return one level sooner.
+ //
+ // This would be the algorithm for updating IsSet (after updating Slots):
+ //
+ // std::atomic<uint64_t> &Bits = IsSet[I.High];
+ // const uint64_t NewBit = 1ULL << I.Low;
+ // uint64_t Old = 0;
+ // while (!Bits.compare_exchange_weak(Old, Old | NewBit))
+ // ;
+
+ // For debugging.
+ unsigned StartBit = 0;
+ unsigned NumBits = 0;
+ friend class llvm::ThreadSafeTrieRawHashMapBase;
+
+public:
+ /// Linked list for ownership of tries. The pointer is owned by TrieSubtrie.
+ std::atomic<TrieSubtrie *> Next;
+
+ /// The (co-allocated) slots of the subtrie.
+ MutableArrayRef<LazyAtomicPointer<TrieNode>> Slots;
+};
+} // end namespace
+
+static size_t getTrieTailSize(size_t StartBit, size_t NumBits) {
+ assert(NumBits < 20 && "Tries should have fewer than ~1M slots");
+ return sizeof(TrieNode *) * (1u << NumBits);
+}
+
+std::unique_ptr<TrieSubtrie> TrieSubtrie::create(size_t StartBit,
+ size_t NumBits) {
+ size_t Size = sizeof(TrieSubtrie) + getTrieTailSize(StartBit, NumBits);
+ void *Memory = ::malloc(Size);
+ TrieSubtrie *S = ::new (Memory) TrieSubtrie(StartBit, NumBits);
+ return std::unique_ptr<TrieSubtrie>(S);
+}
+
+TrieSubtrie::TrieSubtrie(size_t StartBit, size_t NumBits)
+ : TrieNode(true), StartBit(StartBit), NumBits(NumBits), Next(nullptr),
+ Slots(reinterpret_cast<LazyAtomicPointer<TrieNode> *>(
+ reinterpret_cast<char *>(this) + sizeof(TrieSubtrie)),
+ (1u << NumBits)) {
+ for (auto *I = Slots.begin(), *E = Slots.end(); I != E; ++I)
+ new (I) LazyAtomicPointer<TrieNode>(nullptr);
+
+ static_assert(
+ std::is_trivially_destructible<LazyAtomicPointer<TrieNode>>::value,
+ "Expected no work in destructor for TrieNode");
+}
+
+TrieSubtrie *TrieSubtrie::sink(
+ size_t I, TrieContent &Content, size_t NumSubtrieBits, size_t NewI,
+ function_ref<TrieSubtrie *(std::unique_ptr<TrieSubtrie>)> Saver) {
+ assert(NumSubtrieBits > 0);
+ std::unique_ptr<TrieSubtrie> S = create(StartBit + NumBits, NumSubtrieBits);
+
+ assert(NewI < S->Slots.size());
+ S->Slots[NewI].store(&Content);
+
+ TrieNode *ExistingNode = &Content;
+ assert(I < Slots.size());
+ if (Slots[I].compare_exchange_strong(ExistingNode, S.get()))
+ return Saver(std::move(S));
+
+ // Another thread created a subtrie already. Return it and let "S" be
+ // destructed.
+ return cast<TrieSubtrie>(ExistingNode);
+}
+
+struct ThreadSafeTrieRawHashMapBase::ImplType {
+ static std::unique_ptr<ImplType> create(size_t StartBit, size_t NumBits) {
+ size_t Size = sizeof(ImplType) + getTrieTailSize(StartBit, NumBits);
+ void *Memory = ::malloc(Size);
+ ImplType *Impl = ::new (Memory) ImplType(StartBit, NumBits);
+ return std::unique_ptr<ImplType>(Impl);
+ }
+
+ TrieSubtrie *save(std::unique_ptr<TrieSubtrie> S) {
+ assert(!S->Next && "Expected S to a freshly-constructed leaf");
+
+ TrieSubtrie *CurrentHead = nullptr;
+ // Add ownership of "S" to front of the list, so that Root -> S ->
+ // Root.Next. This works by repeatedly setting S->Next to a candidate value
+ // of Root.Next (initially nullptr), then setting Root.Next to S once the
+ // candidate matches reality.
+ while (!Root.Next.compare_exchange_weak(CurrentHead, S.get()))
+ S->Next.exchange(CurrentHead);
+
+ // Ownership transferred to subtrie successfully. Release the unique_ptr.
+ return S.release();
+ }
+
+ static void *operator new(size_t Size) { return ::malloc(Size); }
+ void operator delete(void *Ptr) { ::free(Ptr); }
+
+ /// FIXME: This should take a function that allocates and constructs the
+ /// content lazily (taking the hash as a separate parameter), in case of
+ /// collision.
+ ThreadSafeAllocator<BumpPtrAllocator> ContentAlloc;
+ TrieSubtrie Root; // Must be last! Tail-allocated.
+
+private:
+ ImplType(size_t StartBit, size_t NumBits) : Root(StartBit, NumBits) {}
+};
+
+ThreadSafeTrieRawHashMapBase::ImplType &
+ThreadSafeTrieRawHashMapBase::getOrCreateImpl() {
+ if (ImplType *Impl = ImplPtr.load())
+ return *Impl;
+
+ // Create a new ImplType and store it if another thread doesn't do so first.
+ // If another thread wins this one is destroyed locally.
+ std::unique_ptr<ImplType> Impl = ImplType::create(0, NumRootBits);
+ ImplType *ExistingImpl = nullptr;
+
+ // If the ownership transferred succesfully, release unique_ptr and return
+ // the pointer to the new ImplType.
+ if (ImplPtr.compare_exchange_strong(ExistingImpl, Impl.get()))
+ return *Impl.release();
+
+ // Already created, return the existing ImplType.
+ return *ExistingImpl;
+}
+
+ThreadSafeTrieRawHashMapBase::PointerBase
+ThreadSafeTrieRawHashMapBase::find(ArrayRef<uint8_t> Hash) const {
+ assert(!Hash.empty() && "Uninitialized hash");
+
+ ImplType *Impl = ImplPtr.load();
+ if (!Impl)
+ return PointerBase();
+
+ TrieSubtrie *S = &Impl->Root;
+ IndexGenerator IndexGen{NumRootBits, NumSubtrieBits, Hash};
+ size_t Index = IndexGen.next();
+ while (true) {
+ // Try to set the content.
+ TrieNode *Existing = S->get(Index);
+ if (!Existing)
+ return PointerBase(S, Index, *IndexGen.StartBit);
+
+ // Check for an exact match.
+ if (auto *ExistingContent = dyn_cast<TrieContent>(Existing))
+ return ExistingContent->getHash() == Hash
+ ? PointerBase(ExistingContent->getValuePointer())
+ : PointerBase(S, Index, *IndexGen.StartBit);
+
+ Index = IndexGen.next();
+ S = cast<TrieSubtrie>(Existing);
+ }
+}
+
+ThreadSafeTrieRawHashMapBase::PointerBase ThreadSafeTrieRawHashMapBase::insert(
+ PointerBase Hint, ArrayRef<uint8_t> Hash,
+ function_ref<const uint8_t *(void *Mem, ArrayRef<uint8_t> Hash)>
+ Constructor) {
+ assert(!Hash.empty() && "Uninitialized hash");
+
+ ImplType &Impl = getOrCreateImpl();
+ TrieSubtrie *S = &Impl.Root;
+ IndexGenerator IndexGen{NumRootBits, NumSubtrieBits, Hash};
+ size_t Index;
+ if (Hint.isHint()) {
+ S = static_cast<TrieSubtrie *>(Hint.P);
+ Index = IndexGen.hint(Hint.I, Hint.B);
+ } else {
+ Index = IndexGen.next();
+ }
+
+ while (true) {
+ // Load the node from the slot, allocating and calling the constructor if
+ // the slot is empty.
+ bool Generated = false;
+ TrieNode &Existing = S->Slots[Index].loadOrGenerate([&]() {
+ Generated = true;
+
+ // Construct the value itself at the tail.
+ uint8_t *Memory = reinterpret_cast<uint8_t *>(
+ Impl.ContentAlloc.Allocate(ContentAllocSize, ContentAllocAlign));
+ const uint8_t *HashStorage = Constructor(Memory + ContentOffset, Hash);
+
+ // Construct the TrieContent header, passing in the offset to the hash.
+ TrieContent *Content = ::new (Memory)
+ TrieContent(ContentOffset, Hash.size(), HashStorage - Memory);
+ assert(Hash == Content->getHash() && "Hash not properly initialized");
+ return Content;
+ });
+ // If we just generated it, return it!
+ if (Generated)
+ return PointerBase(cast<TrieContent>(Existing).getValuePointer());
+
+ if (auto *ST = dyn_cast<TrieSubtrie>(&Existing)) {
+ S = ST;
+ Index = IndexGen.next();
+ continue;
+ }
+
+ // Return the existing content if it's an exact match!
+ auto &ExistingContent = cast<TrieContent>(Existing);
+ if (ExistingContent.getHash() == Hash)
+ return PointerBase(ExistingContent.getValuePointer());
+
+ // Sink the existing content as long as the indexes match.
+ while (true) {
+ size_t NextIndex = IndexGen.next();
+ size_t NewIndexForExistingContent =
+ IndexGen.getCollidingBits(ExistingContent.getHash());
+ S = S->sink(Index, ExistingContent, IndexGen.getNumBits(),
+ NewIndexForExistingContent,
+ [&Impl](std::unique_ptr<TrieSubtrie> S) {
+ return Impl.save(std::move(S));
+ });
+ Index = NextIndex;
+
+ // Found the difference.
+ if (NextIndex != NewIndexForExistingContent)
+ break;
+ }
+ }
+}
+
+ThreadSafeTrieRawHashMapBase::ThreadSafeTrieRawHashMapBase(
+ size_t ContentAllocSize, size_t ContentAllocAlign, size_t ContentOffset,
+ std::optional<size_t> NumRootBits, std::optional<size_t> NumSubtrieBits)
+ : ContentAllocSize(ContentAllocSize), ContentAllocAlign(ContentAllocAlign),
+ ContentOffset(ContentOffset),
+ NumRootBits(NumRootBits ? *NumRootBits : DefaultNumRootBits),
+ NumSubtrieBits(NumSubtrieBits ? *NumSubtrieBits : DefaultNumSubtrieBits),
+ ImplPtr(nullptr) {
+ assert((!NumRootBits || *NumRootBits < 20) &&
+ "Root should have fewer than ~1M slots");
+ assert((!NumSubtrieBits || *NumSubtrieBits < 10) &&
+ "Subtries should have fewer than ~1K slots");
+}
+
+ThreadSafeTrieRawHashMapBase::ThreadSafeTrieRawHashMapBase(
+ ThreadSafeTrieRawHashMapBase &&RHS)
+ : ContentAllocSize(RHS.ContentAllocSize),
+ ContentAllocAlign(RHS.ContentAllocAlign),
+ ContentOffset(RHS.ContentOffset), NumRootBits(RHS.NumRootBits),
+ NumSubtrieBits(RHS.NumSubtrieBits) {
+ // Steal the root from RHS.
+ ImplPtr = RHS.ImplPtr.exchange(nullptr);
+}
+
+ThreadSafeTrieRawHashMapBase::~ThreadSafeTrieRawHashMapBase() {
+ assert(!ImplPtr.load() && "Expected subclass to call destroyImpl()");
+}
+
+void ThreadSafeTrieRawHashMapBase::destroyImpl(
+ function_ref<void(void *)> Destructor) {
+ std::unique_ptr<ImplType> Impl(ImplPtr.exchange(nullptr));
+ if (!Impl)
+ return;
+
+ // Destroy content nodes throughout trie. Avoid destroying any subtries since
+ // we need TrieNode::classof() to find the content nodes.
+ //
+ // FIXME: Once we have bitsets (see FIXME in TrieSubtrie class), use them
+ // facilitate sparse iteration here.
+ if (Destructor)
+ for (TrieSubtrie *Trie = &Impl->Root; Trie; Trie = Trie->Next.load())
+ for (auto &Slot : Trie->Slots)
+ if (auto *Content = dyn_cast_or_null<TrieContent>(Slot.load()))
+ Destructor(Content->getValuePointer());
+
+ // Destroy the subtries. Incidentally, this destroys them in the reverse order
+ // of saving.
+ TrieSubtrie *Trie = Impl->Root.Next;
+ while (Trie) {
+ TrieSubtrie *Next = Trie->Next.exchange(nullptr);
+ delete Trie;
+ Trie = Next;
+ }
+}
+
+ThreadSafeTrieRawHashMapBase::PointerBase
+ThreadSafeTrieRawHashMapBase::getRoot() const {
+ ImplType *Impl = ImplPtr.load();
+ if (!Impl)
+ return PointerBase();
+ return PointerBase(&Impl->Root);
+}
+
+unsigned ThreadSafeTrieRawHashMapBase::getStartBit(
+ ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+ assert(!P.isHint() && "Not a valid trie");
+ if (!P.P)
+ return 0;
+ if (auto *S = dyn_cast<TrieSubtrie>((TrieNode *)P.P))
+ return S->StartBit;
+ return 0;
+}
+
+unsigned ThreadSafeTrieRawHashMapBase::getNumBits(
+ ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+ assert(!P.isHint() && "Not a valid trie");
+ if (!P.P)
+ return 0;
+ if (auto *S = dyn_cast<TrieSubtrie>((TrieNode *)P.P))
+ return S->NumBits;
+ return 0;
+}
+
+unsigned ThreadSafeTrieRawHashMapBase::getNumSlotUsed(
+ ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+ assert(!P.isHint() && "Not a valid trie");
+ if (!P.P)
+ return 0;
+ auto *S = dyn_cast<TrieSubtrie>((TrieNode *)P.P);
+ if (!S)
+ return 0;
+ unsigned Num = 0;
+ for (unsigned I = 0, E = S->Slots.size(); I < E; ++I)
+ if (auto *E = S->Slots[I].load())
+ ++Num;
+ return Num;
+}
+
+std::string ThreadSafeTrieRawHashMapBase::getTriePrefixAsString(
+ ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+ assert(!P.isHint() && "Not a valid trie");
+ if (!P.P)
+ return "";
+
+ auto *S = dyn_cast<TrieSubtrie>((TrieNode *)P.P);
+ if (!S || !S->IsSubtrie)
+ return "";
+
+ // Find a TrieContent node which has hash stored. Depth search following the
+ // first used slot until a TrieContent node is found.
+ TrieSubtrie *Current = S;
+ TrieContent *Node = nullptr;
+ while (Current) {
+ TrieSubtrie *Next = nullptr;
+ // find first used slot in the trie.
+ for (unsigned I = 0, E = Current->Slots.size(); I < E; ++I) {
+ auto *S = Current->get(I);
+ if (!S)
+ continue;
+
+ if (auto *Content = dyn_cast<TrieContent>(S))
+ Node = Content;
+ else if (auto *Sub = dyn_cast<TrieSubtrie>(S))
+ Next = Sub;
+ break;
+ }
+
+ // Found the node.
+ if (Node)
+ break;
+
+ // Continue to the next level if the node is not found.
+ Current = Next;
+ }
+
+ assert(Node && "malformed trie, cannot find TrieContent on leaf node");
+ // The prefix for the current trie is the first `StartBit` of the content
+ // stored underneath this subtrie.
+ std::string Str;
+ raw_string_ostream SS(Str);
+
+ unsigned StartFullBytes = (S->StartBit + 1) / 8 - 1;
+ SS << toHex(toStringRef(Node->getHash()).take_front(StartFullBytes),
+ /*LowerCase=*/true);
+
+ // For the part of the prefix that doesn't fill a byte, print raw bit values.
+ std::string Bits;
+ for (unsigned I = StartFullBytes * 8, E = S->StartBit; I < E; ++I) {
+ unsigned Index = I / 8;
+ unsigned Offset = 7 - I % 8;
+ Bits.push_back('0' + ((Node->getHash()[Index] >> Offset) & 1));
+ }
+
+ if (!Bits.empty())
+ SS << "[" << Bits << "]";
+
+ return SS.str();
+}
+
+unsigned ThreadSafeTrieRawHashMapBase::getNumTries() const {
+ ImplType *Impl = ImplPtr.load();
+ if (!Impl)
+ return 0;
+ unsigned Num = 0;
+ for (TrieSubtrie *Trie = &Impl->Root; Trie; Trie = Trie->Next.load())
+ ++Num;
+ return Num;
+}
+
+ThreadSafeTrieRawHashMapBase::PointerBase
+ThreadSafeTrieRawHashMapBase::getNextTrie(
+ ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+ assert(!P.isHint() && "Not a valid trie");
+ if (!P.P)
+ return PointerBase();
+ auto *S = dyn_cast<TrieSubtrie>((TrieNode *)P.P);
+ if (!S)
+ return PointerBase();
+ if (auto *E = S->Next.load())
+ return PointerBase(E);
+ return PointerBase();
+}
diff --git a/llvm/unittests/ADT/CMakeLists.txt b/llvm/unittests/ADT/CMakeLists.txt
index 12d7325036bf051..618f7b0f5ea7858 100644
--- a/llvm/unittests/ADT/CMakeLists.txt
+++ b/llvm/unittests/ADT/CMakeLists.txt
@@ -83,6 +83,7 @@ add_llvm_unittest(ADTTests
StringSetTest.cpp
StringSwitchTest.cpp
TinyPtrVectorTest.cpp
+ TrieRawHashMapTest.cpp
TwineTest.cpp
TypeSwitchTest.cpp
TypeTraitsTest.cpp
diff --git a/llvm/unittests/ADT/TrieRawHashMapTest.cpp b/llvm/unittests/ADT/TrieRawHashMapTest.cpp
new file mode 100644
index 000000000000000..c9081f547812e90
--- /dev/null
+++ b/llvm/unittests/ADT/TrieRawHashMapTest.cpp
@@ -0,0 +1,346 @@
+//===- TrieRawHashMapTest.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/TrieRawHashMap.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/SHA1.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace llvm {
+class TrieRawHashMapTestHelper {
+public:
+ TrieRawHashMapTestHelper() = default;
+
+ void setTrie(ThreadSafeTrieRawHashMapBase *T) { Trie = T; }
+
+ ThreadSafeTrieRawHashMapBase::PointerBase getRoot() const {
+ return Trie->getRoot();
+ }
+ unsigned getStartBit(ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+ return Trie->getStartBit(P);
+ }
+ unsigned getNumBits(ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+ return Trie->getNumBits(P);
+ }
+ unsigned getNumSlotUsed(ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+ return Trie->getNumSlotUsed(P);
+ }
+ unsigned getNumTries() const { return Trie->getNumTries(); }
+ std::string
+ getTriePrefixAsString(ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+ return Trie->getTriePrefixAsString(P);
+ }
+ ThreadSafeTrieRawHashMapBase::PointerBase
+ getNextTrie(ThreadSafeTrieRawHashMapBase::PointerBase P) const {
+ return Trie->getNextTrie(P);
+ }
+
+private:
+ ThreadSafeTrieRawHashMapBase *Trie = nullptr;
+};
+} // namespace llvm
+
+namespace {
+template <typename DataType, size_t HashSize = sizeof(uint64_t)>
+class SimpleTrieHashMapTest : public TrieRawHashMapTestHelper,
+ public ::testing::Test {
+public:
+ using NumType = DataType;
+ using HashType = std::array<uint8_t, HashSize>;
+ using TrieType = ThreadSafeTrieRawHashMap<DataType, sizeof(HashType)>;
+
+ TrieType &createTrie(size_t RootBits, size_t SubtrieBits) {
+ auto &Ret = Trie.emplace(RootBits, SubtrieBits);
+ TrieRawHashMapTestHelper::setTrie(&Ret);
+ return Ret;
+ }
+
+ void destroyTrie() { Trie.reset(); }
+ ~SimpleTrieHashMapTest() { destroyTrie(); }
+
+ // Use the number itself as hash to test the pathological case.
+ static HashType hash(uint64_t Num) {
+ uint64_t HashN =
+ llvm::support::endian::byte_swap(Num, llvm::endianness::big);
+ HashType Hash;
+ memcpy(&Hash[0], &HashN, sizeof(HashType));
+ return Hash;
+ };
+
+private:
+ std::optional<TrieType> Trie;
+};
+
+using SmallNodeTrieTest = SimpleTrieHashMapTest<uint64_t>;
+
+TEST_F(SmallNodeTrieTest, TrieAllocation) {
+ NumType Numbers[] = {
+ 0x0, std::numeric_limits<NumType>::max(), 0x1, 0x2,
+ 0x3, std::numeric_limits<NumType>::max() - 1u,
+ };
+
+ unsigned ExpectedTries[] = {
+ 1, // Allocate Root.
+ 1, // Both on the root.
+ 64, // 0 and 1 sinks all the way down.
+ 64, // no new allocation needed.
+ 65, // need a new node between 2 and 3.
+ 65 + 63, // 63 new allocation to sink two big numbers all the way.
+ };
+
+ const char *ExpectedPrefix[] = {
+ "", // Root.
+ "", // Root.
+ "00000000000000[0000000]",
+ "00000000000000[0000000]",
+ "00000000000000[0000001]",
+ "ffffffffffffff[1111111]",
+ };
+
+ // Use root and subtrie sizes of 1 so this gets sunk quite deep.
+ auto &Trie = createTrie(/*RootBits=*/1, /*SubtrieBits=*/1);
+
+ for (unsigned I = 0; I < 6; ++I) {
+ // Lookup first to exercise hint code for deep tries.
+ TrieType::pointer Lookup = Trie.find(hash(Numbers[I]));
+ EXPECT_FALSE(Lookup);
+
+ Trie.insert(Lookup, TrieType::value_type(hash(Numbers[I]), Numbers[I]));
+ EXPECT_EQ(getNumTries(), ExpectedTries[I]);
+ EXPECT_EQ(getTriePrefixAsString(getNextTrie(getRoot())), ExpectedPrefix[I]);
+ }
+}
+
+TEST_F(SmallNodeTrieTest, TrieStructure) {
+ NumType Numbers[] = {
+ // Three numbers that will nest deeply to test (1) sinking subtries and
+ // (2) deep, non-trivial hints.
+ std::numeric_limits<NumType>::max(),
+ std::numeric_limits<NumType>::max() - 2u,
+ std::numeric_limits<NumType>::max() - 3u,
+ // One number to stay at the top-level.
+ 0x37,
+ };
+
+ // Use root and subtrie sizes of 1 so this gets sunk quite deep.
+ auto &Trie = createTrie(/*RootBits=*/1, /*SubtrieBits=*/1);
+
+ for (NumType N : Numbers) {
+ // Lookup first to exercise hint code for deep tries.
+ TrieType::pointer Lookup = Trie.find(hash(N));
+ EXPECT_FALSE(Lookup);
+
+ Trie.insert(Lookup, TrieType::value_type(hash(N), N));
+ }
+ for (NumType N : Numbers) {
+ TrieType::pointer Lookup = Trie.find(hash(N));
+ EXPECT_TRUE(Lookup);
+ if (!Lookup)
+ continue;
+ EXPECT_EQ(hash(N), Lookup->Hash);
+ EXPECT_EQ(N, Lookup->Data);
+
+ // Confirm a subsequent insertion fails to overwrite by trying to insert a
+ // bad value.
+ auto Result = Trie.insert(Lookup, TrieType::value_type(hash(N), N - 1));
+ EXPECT_EQ(N, Result->Data);
+ }
+
+ // Check the trie so we can confirm the structure is correct. Each subtrie
+ // should have 2 slots. The root's index=0 should have the content for
+ // 0x37 directly, and index=1 should be a linked-list of subtries, finally
+ // ending with content for (max-2) and (max-3).
+ //
+ // Note: This structure is not exhaustive (too expensive to update tests),
+ // but it does test that the dump format is somewhat readable and that the
+ // basic structure is correct.
+ //
+ // Note: This test requires that the trie reads bytes starting from index 0
+ // of the array of uint8_t, and then reads each byte's bits from high to low.
+
+ // Check the Trie.
+ // We should allocated a total of 64 SubTries for 64 bit hash.
+ ASSERT_EQ(getNumTries(), 64u);
+ // Check the root trie. Two slots and both are used.
+ ASSERT_EQ(getNumSlotUsed(getRoot()), 2u);
+ // Check last subtrie.
+ // Last allocated trie is the next node in the allocation chain.
+ auto LastAlloctedSubTrie = getNextTrie(getRoot());
+ ASSERT_EQ(getTriePrefixAsString(LastAlloctedSubTrie),
+ "ffffffffffffff[1111110]");
+ ASSERT_EQ(getStartBit(LastAlloctedSubTrie), 63u);
+ ASSERT_EQ(getNumBits(LastAlloctedSubTrie), 1u);
+ ASSERT_EQ(getNumSlotUsed(LastAlloctedSubTrie), 2u);
+}
+
+TEST_F(SmallNodeTrieTest, TrieStructureSmallFinalSubtrie) {
+ NumType Numbers[] = {
+ // Three numbers that will nest deeply to test (1) sinking subtries and
+ // (2) deep, non-trivial hints.
+ std::numeric_limits<NumType>::max(),
+ std::numeric_limits<NumType>::max() - 2u,
+ std::numeric_limits<NumType>::max() - 3u,
+ // One number to stay at the top-level.
+ 0x37,
+ };
+
+ // Use subtrie size of 5 to avoid hitting 64 evenly, making the final subtrie
+ // small.
+ auto &Trie = createTrie(/*RootBits=*/8, /*SubtrieBits=*/5);
+
+ for (NumType N : Numbers) {
+ // Lookup first to exercise hint code for deep tries.
+ TrieType::pointer Lookup = Trie.find(hash(N));
+ EXPECT_FALSE(Lookup);
+
+ Trie.insert(Lookup, TrieType::value_type(hash(N), N));
+ }
+ for (NumType N : Numbers) {
+ TrieType::pointer Lookup = Trie.find(hash(N));
+ ASSERT_TRUE(Lookup);
+ EXPECT_EQ(hash(N), Lookup->Hash);
+ EXPECT_EQ(N, Lookup->Data);
+
+ // Confirm a subsequent insertion fails to overwrite by trying to insert a
+ // bad value.
+ auto Result = Trie.insert(Lookup, TrieType::value_type(hash(N), N - 1));
+ EXPECT_EQ(N, Result->Data);
+ }
+
+ // Check the trie so we can confirm the structure is correct. The root
+ // should have 2^8=256 slots, most subtries should have 2^5=32 slots, and the
+ // deepest subtrie should have 2^1=2 slots (since (64-8)mod(5)=1).
+ // should have 2 slots. The root's index=0 should have the content for
+ // 0x37 directly, and index=1 should be a linked-list of subtries, finally
+ // ending with content for (max-2) and (max-3).
+ //
+ // Note: This structure is not exhaustive (too expensive to update tests),
+ // but it does test that the dump format is somewhat readable and that the
+ // basic structure is correct.
+ //
+ // Note: This test requires that the trie reads bytes starting from index 0
+ // of the array of uint8_t, and then reads each byte's bits from high to low.
+
+ // Check the Trie.
+ // 64 bit hash = 8 + 5 * 11 + 1, so 1 root, 11 8bit subtrie and 1 last level
+ // subtrie, 13 total.
+ ASSERT_EQ(getNumTries(), 13u);
+ // Check the root trie. Two slots and both are used.
+ ASSERT_EQ(getNumSlotUsed(getRoot()), 2u);
+ // Check last subtrie.
+ // Last allocated trie is the next node in the allocation chain.
+ auto LastAlloctedSubTrie = getNextTrie(getRoot());
+ ASSERT_EQ(getTriePrefixAsString(LastAlloctedSubTrie),
+ "ffffffffffffff[1111110]");
+ ASSERT_EQ(getStartBit(LastAlloctedSubTrie), 63u);
+ ASSERT_EQ(getNumBits(LastAlloctedSubTrie), 1u);
+ ASSERT_EQ(getNumSlotUsed(LastAlloctedSubTrie), 2u);
+}
+
+TEST_F(SmallNodeTrieTest, TrieDestructionLoop) {
+ // Test destroying large Trie. Make sure there is no recursion that can
+ // overflow the stack.
+
+ // Limit the tries to 2 slots (1 bit) to generate subtries at a higher rate.
+ auto &Trie = createTrie(/*NumRootBits=*/1, /*NumSubtrieBits=*/1);
+
+ // Fill them up. Pick a MaxN high enough to cause a stack overflow in debug
+ // builds.
+ static constexpr uint64_t MaxN = 100000;
+ for (uint64_t N = 0; N != MaxN; ++N) {
+ HashType Hash = hash(N);
+ Trie.insert(TrieType::pointer(), TrieType::value_type(Hash, NumType{N}));
+ }
+
+ // Destroy tries. If destruction is recursive and MaxN is high enough, these
+ // will both fail.
+ destroyTrie();
+}
+
+struct NumWithDestructorT {
+ uint64_t Num;
+ llvm::function_ref<void()> DestructorCallback;
+ ~NumWithDestructorT() { DestructorCallback(); }
+};
+
+using NodeWithDestructorTrieTest = SimpleTrieHashMapTest<NumWithDestructorT>;
+
+TEST_F(NodeWithDestructorTrieTest, TrieDestructionLoop) {
+ // Test destroying large Trie. Make sure there is no recursion that can
+ // overflow the stack.
+
+ // Limit the tries to 2 slots (1 bit) to generate subtries at a higher rate.
+ auto &Trie = createTrie(/*NumRootBits=*/1, /*NumSubtrieBits=*/1);
+
+ // Fill them up. Pick a MaxN high enough to cause a stack overflow in debug
+ // builds.
+ static constexpr uint64_t MaxN = 100000;
+
+ uint64_t DestructorCalled = 0;
+ auto DtorCallback = [&DestructorCalled]() { ++DestructorCalled; };
+ for (uint64_t N = 0; N != MaxN; ++N) {
+ HashType Hash = hash(N);
+ Trie.insert(TrieType::pointer(),
+ TrieType::value_type(Hash, NumType{N, DtorCallback}));
+ }
+ // Reset the count after all the temporaries get destroyed.
+ DestructorCalled = 0;
+
+ // Destroy tries. If destruction is recursive and MaxN is high enough, these
+ // will both fail.
+ destroyTrie();
+
+ // Count the number of destructor calls during `destroyTrie()`.
+ ASSERT_EQ(DestructorCalled, MaxN);
+}
+
+using NumStrNodeTrieTest = SimpleTrieHashMapTest<std::string>;
+
+TEST_F(NumStrNodeTrieTest, TrieInsertLazy) {
+ for (unsigned RootBits : {2, 3, 6, 10}) {
+ for (unsigned SubtrieBits : {2, 3, 4}) {
+ auto &Trie = createTrie(RootBits, SubtrieBits);
+ for (int I = 0, E = 1000; I != E; ++I) {
+ TrieType::pointer Lookup;
+ HashType H = hash(I);
+ if (I & 1)
+ Lookup = Trie.find(H);
+
+ auto insertNum = [&](uint64_t Num) {
+ std::string S = Twine(I).str();
+ auto Hash = hash(Num);
+ return Trie.insertLazy(
+ Hash, [&](TrieType::LazyValueConstructor C) { C(std::move(S)); });
+ };
+ auto S1 = insertNum(I);
+ // The address of the Data should be the same.
+ EXPECT_EQ(&S1->Data, &insertNum(I)->Data);
+
+ auto insertStr = [&](std::string S) {
+ int Num = std::stoi(S);
+ return insertNum(Num);
+ };
+ std::string S2 = S1->Data;
+ // The address of the Data should be the same.
+ EXPECT_EQ(&S1->Data, &insertStr(S2)->Data);
+ }
+ for (int I = 0, E = 1000; I != E; ++I) {
+ std::string S = Twine(I).str();
+ TrieType::pointer Lookup = Trie.find(hash(I));
+ EXPECT_TRUE(Lookup);
+ if (!Lookup)
+ continue;
+ EXPECT_EQ(S, Lookup->Data);
+ }
+ }
+ }
+}
+} // end anonymous namespace
More information about the llvm-commits
mailing list