[llvm] 8e6f260 - [DenseMap] Share rehash and grow for relocatable bucket types. NFC (#225018)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 27 12:14:53 PDT 2026
Author: Fangrui Song
Date: 2026-09-27T12:14:42-07:00
New Revision: 8e6f2603dc5d4a93400aa8ba28ea624644be0c9a
URL: https://github.com/llvm/llvm-project/commit/8e6f2603dc5d4a93400aa8ba28ea624644be0c9a
DIFF: https://github.com/llvm/llvm-project/commit/8e6f2603dc5d4a93400aa8ba28ea624644be0c9a.diff
LOG: [DenseMap] Share rehash and grow for relocatable bucket types. NFC (#225018)
DenseMap is heavily instantiated and moveFrom and grow are among the
largest code families. For trivially copy constructible and destructible
bucket types (also satisfied by std::pair), call an out-of-line
`growRelocatable` with size/align/hasher (dictionary passing style).
A pointer key hashed by its value hashes inline, which a null
BucketHasher asks for, sparing a call per entry.
DenseMapInfo<T *>::PointerValueHash names the class declaring it, so an
info that specializes or inherits it to hash the pointee -- Attributor's
InstExclusionSet, VPCSEDenseMapInfo, MachineInstrExpressionTrait --
keeps its own hash. Any other key hashes through a thunk, one per
(KeyT, KeyInfoT) whatever the map's value type.
SmallDenseMap's in-place rehash for remove_if shares the loop through
`rehashRelocatable`. It rehashes once rather than into a temporary map
and back, which can reorder a probe chain that wraps around the table.
Building clang takes 0.86% fewer instructions and the binary is 0.64%
smaller.
Aided by Opus 5
Added:
llvm/lib/Support/DenseMap.cpp
Modified:
llvm/include/llvm/ADT/DenseMap.h
llvm/include/llvm/ADT/DenseMapInfo.h
llvm/lib/Support/CMakeLists.txt
llvm/unittests/ADT/DenseMapTest.cpp
llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
Removed:
################################################################################
diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h
index 56c56c489915e..f1763dcfed62a 100644
--- a/llvm/include/llvm/ADT/DenseMap.h
+++ b/llvm/include/llvm/ADT/DenseMap.h
@@ -111,6 +111,9 @@ inline void setUsed(UsedT *U, size_t I) { U[I >> 5] |= UsedT(1) << (I & 31); }
inline void unsetUsed(UsedT *U, size_t I) {
U[I >> 5] &= ~(UsedT(1) << (I & 31));
}
+inline void clearUsed(UsedT *U, unsigned Num) {
+ std::memset(U, 0, usedWords(Num) * sizeof(UsedT));
+}
// Invoke Func(I) for each occupied bucket index I in [0, N). Set always_inline;
// otherwise, for a heavy caller such as moveFrom's rehash, the inliner can
@@ -134,10 +137,60 @@ LLVM_ATTRIBUTE_ALWAYS_INLINE void forEachUsed(const UsedT *U, unsigned N,
template <typename BucketT> constexpr size_t allocAlign() {
return std::max(alignof(BucketT), alignof(UsedT));
}
+inline size_t allocBytes(size_t BucketSize, unsigned Num) {
+ return BucketSize * static_cast<size_t>(Num) + usedWords(Num) * sizeof(UsedT);
+}
template <typename BucketT> size_t allocBytes(unsigned Num) {
- return sizeof(BucketT) * static_cast<size_t>(Num) +
- usedWords(Num) * sizeof(UsedT);
+ return allocBytes(sizeof(BucketT), Num);
}
+inline UsedT *usedFor(void *Buckets, size_t BucketSize, unsigned Num) {
+ assert(BucketSize * static_cast<size_t>(Num) % alignof(UsedT) == 0 &&
+ "used array would be misaligned");
+ return reinterpret_cast<UsedT *>(static_cast<char *>(Buckets) +
+ BucketSize * static_cast<size_t>(Num));
+}
+
+/// Hashes the key, at offset 0 in a bucket. Null asks the out-of-line rehash
+/// loop in DenseMap.cpp to inline the pointer hash.
+using BucketHasher = unsigned (*)(const void *Key);
+
+// True when the map hashes a pointer key by its value: the primary
+// DenseMapInfo<T *> declares PointerValueHash naming itself, which a
+// specialization or derived info for one pointer type does not. Only the
+// primary is asked, since GCC without DR1170 errors on a lookup that reaches a
+// private base, as clang::WeakInfo's info has.
+template <typename KeyT, typename KeyInfoT, typename = void>
+inline constexpr bool hashesPointerValue = false;
+template <typename T>
+inline constexpr bool hashesPointerValue<
+ T *, DenseMapInfo<T *>,
+ std::enable_if_t<std::is_same_v<
+ typename DenseMapInfo<T *>::PointerValueHash, DenseMapInfo<T *>>>> =
+ true;
+
+// Kept out of DenseMapBase so that map types sharing a key share one thunk.
+template <typename KeyT, typename KeyInfoT> constexpr BucketHasher hasherFor() {
+ if constexpr (hashesPointerValue<KeyT, KeyInfoT>)
+ return nullptr;
+ else
+ return [](const void *Key) -> unsigned {
+ return KeyInfoT::getHashValue(*static_cast<const KeyT *>(Key));
+ };
+}
+
+/// Rehash the live buckets of \p Src into the empty \p Dst, which must have
+/// room for all of them.
+LLVM_ABI void rehashRelocatable(void *Dst, UsedT *DstUsed,
+ unsigned DstNumBuckets, const void *Src,
+ const UsedT *SrcUsed, unsigned SrcNumBuckets,
+ size_t BucketSize, BucketHasher Hasher);
+
+/// Allocate a table of \p NewNumBuckets buckets and rehash the \p OldNumBuckets
+/// buckets at \p OldBuckets into it, freeing them if \p FreeOld.
+LLVM_ABI void *growRelocatable(void *OldBuckets, const UsedT *OldUsed,
+ unsigned OldNumBuckets, unsigned NewNumBuckets,
+ size_t BucketSize, size_t Align,
+ BucketHasher Hasher, bool FreeOld);
} // namespace densemap::detail
@@ -228,9 +281,7 @@ class DenseMapBase : public DebugEpochBase {
}
destroyAll();
- std::memset(getUsed(), 0,
- llvm::densemap::detail::usedWords(getNumBuckets()) *
- sizeof(UsedT));
+ llvm::densemap::detail::clearUsed(getUsed(), getNumBuckets());
setNumEntries(0);
}
@@ -535,11 +586,8 @@ class DenseMapBase : public DebugEpochBase {
assert((getNumBuckets() & (getNumBuckets() - 1)) == 0 &&
"# initial buckets must be a power of two!");
- if (getNumBuckets()) {
- std::memset(getUsed(), 0,
- llvm::densemap::detail::usedWords(getNumBuckets()) *
- sizeof(UsedT));
- }
+ if (getNumBuckets())
+ llvm::densemap::detail::clearUsed(getUsed(), getNumBuckets());
}
/// Returns the number of buckets to allocate to ensure that the DenseMap can
@@ -553,6 +601,10 @@ class DenseMapBase : public DebugEpochBase {
return NextPowerOf2(NumEntries * 4 / 3 + 1);
}
+ static constexpr llvm::densemap::detail::BucketHasher hasher() {
+ return llvm::densemap::detail::hasherFor<KeyT, KeyInfoT>();
+ }
+
// Move key/value from Other to *this.
// Other is left in a valid but empty state.
LLVM_ATTRIBUTE_NOINLINE void moveFrom(DerivedT &Other) {
@@ -731,13 +783,19 @@ class DenseMapBase : public DebugEpochBase {
}
LLVM_ATTRIBUTE_NOINLINE void grow(unsigned MinNumBuckets) {
- unsigned NumBuckets = DerivedT::roundUpNumBuckets(MinNumBuckets);
- DerivedT Tmp(NumBuckets, ExactBucketCount{});
- Tmp.moveFrom(derived());
- if (derived().maybeMoveFast(std::move(Tmp)))
- return;
- initWithExactBucketCount(NumBuckets);
- moveFrom(Tmp);
+ assert((MinNumBuckets == 0 || isPowerOf2_32(MinNumBuckets)) &&
+ "bucket count must be zero or a power of two");
+ if constexpr (llvm::densemap::detail::isRelocatableBucket<BucketT>) {
+ derived().growShared(MinNumBuckets);
+ } else {
+ unsigned NumBuckets = DerivedT::roundUpNumBuckets(MinNumBuckets);
+ DerivedT Tmp(NumBuckets, ExactBucketCount{});
+ Tmp.moveFrom(derived());
+ if (derived().maybeMoveFast(std::move(Tmp)))
+ return;
+ initWithExactBucketCount(NumBuckets);
+ moveFrom(Tmp);
+ }
}
template <typename LookupKeyT>
@@ -951,6 +1009,21 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
typename BaseT::Rep getRep() const { return {Buckets, Used, NumBuckets}; }
+ void setStorage(void *Storage, unsigned Num) {
+ Buckets = static_cast<BucketT *>(Storage);
+ Used = llvm::densemap::detail::usedFor(Storage, sizeof(BucketT), Num);
+ NumBuckets = Num;
+ }
+
+ void growShared(unsigned MinNumBuckets) {
+ unsigned NewNumBuckets = roundUpNumBuckets(MinNumBuckets);
+ setStorage(llvm::densemap::detail::growRelocatable(
+ Buckets, Used, NumBuckets, NewNumBuckets, sizeof(BucketT),
+ llvm::densemap::detail::allocAlign<BucketT>(),
+ BaseT::hasher(), /*FreeOld=*/true),
+ NewNumBuckets);
+ }
+
UsedT *getUsed() const { return Used; }
unsigned getNumBuckets() const { return NumBuckets; }
@@ -967,22 +1040,15 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
}
bool allocateBuckets(unsigned Num) {
- NumBuckets = Num;
- if (NumBuckets == 0) {
+ if (Num == 0) {
Buckets = nullptr;
Used = nullptr;
+ NumBuckets = 0;
return false;
}
-
- auto *Storage = static_cast<char *>(
- allocate_buffer(llvm::densemap::detail::allocBytes<BucketT>(NumBuckets),
- llvm::densemap::detail::allocAlign<BucketT>()));
- Buckets = reinterpret_cast<BucketT *>(Storage);
- // NumBuckets is a power of two >= 4 (getMinBucketToReserveForEntries(1) is
- // 4), so the used array trailing the buckets is aligned.
- assert(sizeof(BucketT) * NumBuckets % alignof(UsedT) == 0 &&
- "used array would be misaligned");
- Used = reinterpret_cast<UsedT *>(Storage + sizeof(BucketT) * NumBuckets);
+ setStorage(allocate_buffer(llvm::densemap::detail::allocBytes<BucketT>(Num),
+ llvm::densemap::detail::allocAlign<BucketT>()),
+ Num);
return true;
}
@@ -991,8 +1057,7 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
void kill() { deallocateBuckets(); }
static unsigned roundUpNumBuckets(unsigned MinNumBuckets) {
- return std::max(64u,
- static_cast<unsigned>(NextPowerOf2(MinNumBuckets - 1)));
+ return std::max(64u, MinNumBuckets);
}
bool maybeMoveFast(DenseMap &&Other) {
@@ -1229,6 +1294,32 @@ class SmallDenseMap
return Small ? InlineBuckets : storage.Large.NumBuckets;
}
+ void setLarge(void *Storage, unsigned NumBuckets) {
+ Small = false;
+ storage.Large = {
+ static_cast<BucketT *>(Storage),
+ llvm::densemap::detail::usedFor(Storage, sizeof(BucketT), NumBuckets),
+ NumBuckets};
+ }
+
+ void growShared(unsigned MinNumBuckets) {
+ unsigned NewNumBuckets = roundUpNumBuckets(MinNumBuckets);
+ // remove_if asks for the count it already has: rehash in place.
+ if (Small && NewNumBuckets <= InlineBuckets) {
+ InlineRep Old = storage.Inline;
+ llvm::densemap::detail::clearUsed(getInlineUsed(), InlineBuckets);
+ llvm::densemap::detail::rehashRelocatable(
+ getInlineBuckets(), getInlineUsed(), InlineBuckets, Old.Buckets,
+ Old.Used, InlineBuckets, sizeof(BucketT), BaseT::hasher());
+ return;
+ }
+ void *Storage = llvm::densemap::detail::growRelocatable(
+ getBuckets(), getUsed(), getNumBuckets(), NewNumBuckets,
+ sizeof(BucketT), llvm::densemap::detail::allocAlign<BucketT>(),
+ BaseT::hasher(), /*FreeOld=*/!Small);
+ setLarge(Storage, NewNumBuckets);
+ }
+
void deallocateBuckets() {
// Fast path in case storage.Large.NumBuckets == 0, just like destroyAll.
// This path is used to destruct zombie instances after moves.
@@ -1247,13 +1338,9 @@ class SmallDenseMap
Small = true;
return true;
}
- Small = false;
- auto *S = static_cast<char *>(
- allocate_buffer(llvm::densemap::detail::allocBytes<BucketT>(Num),
- llvm::densemap::detail::allocAlign<BucketT>()));
- storage.Large.Buckets = reinterpret_cast<BucketT *>(S);
- storage.Large.Used = reinterpret_cast<UsedT *>(S + sizeof(BucketT) * Num);
- storage.Large.NumBuckets = Num;
+ setLarge(allocate_buffer(llvm::densemap::detail::allocBytes<BucketT>(Num),
+ llvm::densemap::detail::allocAlign<BucketT>()),
+ Num);
return true;
}
@@ -1267,8 +1354,7 @@ class SmallDenseMap
static unsigned roundUpNumBuckets(unsigned MinNumBuckets) {
if (MinNumBuckets <= InlineBuckets)
return InlineBuckets;
- return std::max(64u,
- static_cast<unsigned>(NextPowerOf2(MinNumBuckets - 1)));
+ return std::max(64u, MinNumBuckets);
}
bool maybeMoveFast(SmallDenseMap &&Other) {
diff --git a/llvm/include/llvm/ADT/DenseMapInfo.h b/llvm/include/llvm/ADT/DenseMapInfo.h
index 3a3dfa9a185b1..575001cd22967 100644
--- a/llvm/include/llvm/ADT/DenseMapInfo.h
+++ b/llvm/include/llvm/ADT/DenseMapInfo.h
@@ -56,6 +56,9 @@ template <typename T, typename Enable = void> struct DenseMapInfo {
// Provide DenseMapInfo for all pointers. Avoid requiring T to be complete so
// clients can instantiate DenseMap<T*, ...> with forward declared key types.
template <typename T> struct DenseMapInfo<T *> {
+ // Select InlinePtrHash in DenseMap.cpp.
+ using PointerValueHash = DenseMapInfo;
+
static unsigned getHashValue(const T *PtrVal) {
return densemap::detail::mix(reinterpret_cast<uintptr_t>(PtrVal));
}
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index df8a2a67ed32c..65ec233299495 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -198,6 +198,7 @@ add_llvm_component_library(LLVMSupport
DebugCounter.cpp
DeltaAlgorithm.cpp
DeltaTree.cpp
+ DenseMap.cpp
DivisionByConstantInfo.cpp
DAGDeltaAlgorithm.cpp
DJB.cpp
diff --git a/llvm/lib/Support/DenseMap.cpp b/llvm/lib/Support/DenseMap.cpp
new file mode 100644
index 0000000000000..eef05bee80a04
--- /dev/null
+++ b/llvm/lib/Support/DenseMap.cpp
@@ -0,0 +1,101 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Support/MemAlloc.h"
+#include <cstring>
+
+using namespace llvm;
+using namespace llvm::densemap;
+using namespace llvm::densemap::detail;
+
+// A nonzero FixedSize turns the bucket copy into a couple of stores.
+template <size_t FixedSize, bool InlinePtrHash>
+static void rehashLoop(void *DstBuckets, UsedT *DstUsed, unsigned Mask,
+ const void *SrcBuckets, const UsedT *SrcUsed,
+ unsigned SrcNumBuckets, size_t RuntimeSize,
+ BucketHasher Hasher) {
+ const size_t BucketSize = FixedSize ? FixedSize : RuntimeSize;
+ char *Dst = static_cast<char *>(DstBuckets);
+ const char *Src = static_cast<const char *>(SrcBuckets);
+ forEachUsed(SrcUsed, SrcNumBuckets, [&](unsigned I) {
+ const char *SrcBucket = Src + static_cast<size_t>(I) * BucketSize;
+ unsigned Hash;
+ if constexpr (InlinePtrHash) {
+ void *Key;
+ std::memcpy(&Key, SrcBucket, sizeof(Key));
+ Hash = DenseMapInfo<void *>::getHashValue(Key);
+ } else {
+ Hash = Hasher(SrcBucket);
+ }
+ unsigned BucketNo = Hash & Mask;
+ while (used(DstUsed, BucketNo))
+ BucketNo = (BucketNo + 1) & Mask;
+ std::memcpy(Dst + static_cast<size_t>(BucketNo) * BucketSize, SrcBucket,
+ BucketSize);
+ setUsed(DstUsed, BucketNo);
+ });
+}
+
+template <bool InlinePtrHash>
+static void rehashBySize(void *Dst, UsedT *DstUsed, unsigned Mask,
+ const void *Src, const UsedT *SrcUsed,
+ unsigned SrcNumBuckets, size_t BucketSize,
+ BucketHasher Hasher) {
+ // The bucket sizes of 95% of the grow instantiations in an LLVM build.
+ switch (BucketSize) {
+#define REHASH_CASE(N) \
+ case N: \
+ return rehashLoop<N, InlinePtrHash>(Dst, DstUsed, Mask, Src, SrcUsed, \
+ SrcNumBuckets, BucketSize, Hasher);
+ REHASH_CASE(4)
+ REHASH_CASE(8)
+ REHASH_CASE(12)
+ REHASH_CASE(16)
+ REHASH_CASE(24)
+ REHASH_CASE(32)
+ REHASH_CASE(40)
+ REHASH_CASE(48)
+#undef REHASH_CASE
+ default:
+ return rehashLoop<0, InlinePtrHash>(Dst, DstUsed, Mask, Src, SrcUsed,
+ SrcNumBuckets, BucketSize, Hasher);
+ }
+}
+
+void densemap::detail::rehashRelocatable(void *Dst, UsedT *DstUsed,
+ unsigned DstNumBuckets,
+ const void *Src, const UsedT *SrcUsed,
+ unsigned SrcNumBuckets,
+ size_t BucketSize,
+ BucketHasher Hasher) {
+ const unsigned Mask = DstNumBuckets - 1;
+ if (!Hasher)
+ return rehashBySize<true>(Dst, DstUsed, Mask, Src, SrcUsed, SrcNumBuckets,
+ BucketSize, Hasher);
+ return rehashBySize<false>(Dst, DstUsed, Mask, Src, SrcUsed, SrcNumBuckets,
+ BucketSize, Hasher);
+}
+
+void *densemap::detail::growRelocatable(void *OldBuckets, const UsedT *OldUsed,
+ unsigned OldNumBuckets,
+ unsigned NewNumBuckets,
+ size_t BucketSize, size_t Align,
+ BucketHasher Hasher, bool FreeOld) {
+ void *Storage = allocate_buffer(allocBytes(BucketSize, NewNumBuckets), Align);
+ UsedT *NewUsed = usedFor(Storage, BucketSize, NewNumBuckets);
+ clearUsed(NewUsed, NewNumBuckets);
+ if (OldNumBuckets) {
+ rehashRelocatable(Storage, NewUsed, NewNumBuckets, OldBuckets, OldUsed,
+ OldNumBuckets, BucketSize, Hasher);
+ if (FreeOld)
+ deallocate_buffer(OldBuckets, allocBytes(BucketSize, OldNumBuckets),
+ Align);
+ }
+ return Storage;
+}
diff --git a/llvm/unittests/ADT/DenseMapTest.cpp b/llvm/unittests/ADT/DenseMapTest.cpp
index 79ed6731b872a..bd49c87d5054f 100644
--- a/llvm/unittests/ADT/DenseMapTest.cpp
+++ b/llvm/unittests/ADT/DenseMapTest.cpp
@@ -10,6 +10,7 @@
#include "CountCopyAndMove.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/DenseMapInfoVariant.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/STLForwardCompat.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/StringRef.h"
@@ -959,6 +960,37 @@ struct B : public A {
struct AlwaysEqType {
bool operator==(const AlwaysEqType &RHS) const { return true; }
};
+
+struct PointeeHashed {
+ unsigned V;
+};
+
+struct ExprKey {
+ unsigned V;
+};
+
+struct AliasedKey {
+ const unsigned *A;
+};
+
+// Shaped like clang::WeakInfo::DenseMapInfoByAliasOnly: an info that privately
+// inherits the pointer info, hiding the tag from a lookup through it.
+struct AliasedKeyInfo : private DenseMapInfo<const unsigned *> {
+ static unsigned getHashValue(const AliasedKey &K) {
+ return DenseMapInfo::getHashValue(K.A);
+ }
+ static bool isEqual(const AliasedKey &L, const AliasedKey &R) {
+ return DenseMapInfo::isEqual(L.A, R.A);
+ }
+};
+
+// Shaped like VPCSEDenseMapInfo: inherits the pointer info, hashes the pointee.
+struct ExprKeyTrait : DenseMapInfo<ExprKey *> {
+ static unsigned getHashValue(const ExprKey *P) { return P->V * 37; }
+ static bool isEqual(const ExprKey *LHS, const ExprKey *RHS) {
+ return LHS->V == RHS->V;
+ }
+};
} // namespace
namespace llvm {
@@ -977,6 +1009,15 @@ template <> struct DenseMapInfo<AlwaysEqType> {
return false;
}
};
+
+// Shaped like Attributor's InstExclusionSet: a specialization for one pointer
+// type that hashes the pointee.
+template <> struct DenseMapInfo<PointeeHashed *> {
+ static unsigned getHashValue(const PointeeHashed *P) { return P->V * 37; }
+ static bool isEqual(const PointeeHashed *LHS, const PointeeHashed *RHS) {
+ return LHS->V == RHS->V;
+ }
+};
} // namespace llvm
namespace {
@@ -1256,4 +1297,107 @@ TEST(DenseMapCustomTest, MoveAssignInvalidatesIterators) {
}
#endif
+// A wrong bucket size or hash puts entries where a later lookup will not probe.
+template <unsigned NumWords> struct RehashValue {
+ uint64_t V[NumWords];
+ RehashValue(unsigned X = 0) : V{X} {}
+ bool operator==(const RehashValue &O) const { return V[0] == O.V[0]; }
+};
+
+static void *rehashTestKey(unsigned I) {
+ return reinterpret_cast<void *>(static_cast<uintptr_t>(I) * 4096 + 16);
+}
+
+// Enough entries that the destination probe walk runs.
+static constexpr unsigned NumRehashKeys = 200;
+
+// Insert Key(1..NumRehashKeys), grow, and look each up again.
+template <typename ContainerT, typename KeyFn = void *(*)(unsigned)>
+static void expectSurvivesRehash(KeyFn Key = rehashTestKey) {
+ constexpr bool IsSet = std::is_same_v<typename ContainerT::key_type,
+ typename ContainerT::value_type>;
+ ContainerT C;
+ for (unsigned I = 1; I <= NumRehashKeys; ++I) {
+ if constexpr (IsSet)
+ C.insert(Key(I));
+ else
+ C[Key(I)] = typename ContainerT::mapped_type(I);
+ }
+ C.reserve(1024);
+ ASSERT_EQ(NumRehashKeys, C.size());
+ for (unsigned I = 1; I <= NumRehashKeys; ++I) {
+ if constexpr (IsSet)
+ EXPECT_TRUE(C.contains(Key(I)));
+ else
+ EXPECT_EQ(typename ContainerT::mapped_type(I), C.lookup(Key(I)));
+ }
+ EXPECT_FALSE(C.contains(Key(NumRehashKeys + 1)));
+}
+
+// As LazyValueInfo's DenseSet<LVIValueHandle, DenseMapInfo<Value *>> does: an
+// info for a type the key converts to, with bookkeeping before the pointer.
+struct RehashConvertibleKey {
+ uint64_t Bookkeeping;
+ void *P;
+ RehashConvertibleKey(void *P = nullptr) : Bookkeeping(~0ULL), P(P) {}
+ operator void *() const { return P; }
+};
+
+TEST(DenseMapCustomTest, LookupSurvivesRehash) {
+ expectSurvivesRehash<DenseSet<void *>>(); // 8-byte bucket
+ expectSurvivesRehash<DenseMap<void *, unsigned>>(); // 16
+ expectSurvivesRehash<DenseMap<void *, RehashValue<2>>>(); // 24
+ expectSurvivesRehash<DenseMap<void *, RehashValue<3>>>(); // 32
+ expectSurvivesRehash<DenseMap<void *, RehashValue<4>>>(); // 40
+ expectSurvivesRehash<DenseMap<void *, RehashValue<5>>>(); // 48
+ expectSurvivesRehash<DenseMap<void *, RehashValue<7>>>(); // 64, runtime size
+ // Spilling the inline buffer rehashes it without freeing it.
+ expectSurvivesRehash<SmallDenseMap<void *, unsigned, 4>>();
+ expectSurvivesRehash<SmallDenseMap<void *, RehashValue<7>, 4>>();
+ expectSurvivesRehash<DenseSet<RehashConvertibleKey, DenseMapInfo<void *>>>();
+}
+
+// A pointer key hashed by its pointee, through a specialization or an info
+// inheriting the pointer info, keeps that hash; an info privately inheriting
+// the pointer info still compiles.
+TEST(DenseMapCustomTest, PointerKeyInfos) {
+ PointeeHashed Specialized[NumRehashKeys + 1];
+ ExprKey Inherited[NumRehashKeys + 1];
+ unsigned Aliased[NumRehashKeys + 1];
+ for (unsigned I = 0; I <= NumRehashKeys; ++I)
+ Specialized[I].V = Inherited[I].V = Aliased[I] = I + 1;
+ expectSurvivesRehash<DenseMap<PointeeHashed *, unsigned>>(
+ [&](unsigned I) { return &Specialized[I - 1]; });
+ expectSurvivesRehash<DenseMap<ExprKey *, unsigned, ExprKeyTrait>>(
+ [&](unsigned I) { return &Inherited[I - 1]; });
+ expectSurvivesRehash<DenseMap<AliasedKey, unsigned, AliasedKeyInfo>>(
+ [&](unsigned I) { return AliasedKey{&Aliased[I - 1]}; });
+}
+
+// remove_if rehashes the inline buckets in place. Five entries in eight buckets
+// leave clusters that wrap, so a survivor needs the probe chain walked.
+TEST(DenseMapCustomTest, SmallRemoveIf) {
+ for (unsigned Trial = 0; Trial != 200; ++Trial) {
+ SmallDenseMap<void *, unsigned, 8> Map;
+ const unsigned First = Trial * 5 + 1;
+ for (unsigned I = First; I != First + 5; ++I)
+ Map[rehashTestKey(I)] = I;
+ ASSERT_EQ(5u, Map.size());
+
+ EXPECT_TRUE(Map.remove_if([](const auto &E) { return E.second % 2 == 0; }));
+ for (unsigned I = First; I != First + 5; ++I) {
+ if (I % 2)
+ EXPECT_EQ(I, Map.lookup(rehashTestKey(I)));
+ else
+ EXPECT_FALSE(Map.contains(rehashTestKey(I)));
+ }
+
+ // Refilling past the inline buckets spills the compacted table.
+ for (unsigned I = 1; I <= NumRehashKeys; ++I)
+ Map[rehashTestKey(I)] = I;
+ for (unsigned I = 1; I <= NumRehashKeys; ++I)
+ EXPECT_EQ(I, Map.lookup(rehashTestKey(I)));
+ }
+}
+
} // namespace
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 37dc6744107ac..3be959fea995f 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -81,6 +81,7 @@ static_library("Support") {
"DebugCounter.cpp",
"DeltaAlgorithm.cpp",
"DeltaTree.cpp",
+ "DenseMap.cpp",
"DivisionByConstantInfo.cpp",
"Driver.cpp",
"DynamicAPInt.cpp",
More information about the llvm-commits
mailing list