[llvm] [DenseMap] Store occupancy in a packed used-bit array (PR #201281)
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 3 06:51:59 PDT 2026
https://github.com/MaskRay updated https://github.com/llvm/llvm-project/pull/201281
>From a7f31d72f015c62f024cbed2ded42ea8f7fa664d Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sun, 31 May 2026 21:23:52 -0700
Subject: [PATCH 1/2] [DenseMap] Store occupancy in a packed used-bit array
Track bucket occupancy in a packed 1-bit-per-bucket "used" array
(uint32_t words) instead of an `Empty` sentinel key. The buckets and the
used array share one allocation. The probing scheme is unchanged.
(uint64_t words lead to slightly larger clang binary.)
Empty buckets are now raw storage: insert placement-news the key/value
and erase destroys them. Any key value is therefore storable --
`DenseMap<unsigned, X>` can use all keys -- which lets `DenseMapInfo`
drop `getEmptyKey` in a follow-up.
This helps find-miss and iteration (the empty terminus and the empty
buckets become a bit test, not a bucket load; for large keys it also
skips the structural compare against the empty key) and large-bucket
insert. It costs find-hit (the matched bucket is loaded either way, so
the bit is pure overhead) and, for small buckets, fill/insert (the
per-insert used-bit write).
The net is a slight stage2 compile-time regression (instructions:u
+0.10% -O3, +0.24% -O0-g); the find-miss and iteration gains are cache
effects an instruction-count metric does not see.
The regression concentrates in hot (pointer key, sizeof(bucket) <= 16)
instantiations; a follow-up may move those to a separate pointer-keyed
container that keeps the in-band sentinel, reclaiming the loss.
getMemorySize now also counts the used array; the InitSize tests are updated and
the BitVectorTest DenseSet test no longer expects inserting a default key to
abort.
Aided by Claude Opus 4.8
---
llvm/docs/ProgrammersManual.rst | 4 +-
llvm/include/llvm/ADT/DenseMap.h | 578 +++++++++++++++++----------
llvm/unittests/ADT/BitVectorTest.cpp | 9 +-
llvm/unittests/ADT/DenseMapTest.cpp | 30 +-
4 files changed, 384 insertions(+), 237 deletions(-)
diff --git a/llvm/docs/ProgrammersManual.rst b/llvm/docs/ProgrammersManual.rst
index ce786a5b7b681..ddb9c57ad4e2a 100644
--- a/llvm/docs/ProgrammersManual.rst
+++ b/llvm/docs/ProgrammersManual.rst
@@ -2170,7 +2170,7 @@ copy-construction, which :ref:`SmallSet <dss_smallset>` and :ref:`SmallPtrSet
llvm/ADT/DenseSet.h
^^^^^^^^^^^^^^^^^^^
-``DenseSet`` is a simple quadratically probed hash table. It excels at supporting
+``DenseSet`` is a simple linearly probed hash table. It excels at supporting
small values: it uses a single allocation to hold all of the pairs that are
currently inserted in the set. ``DenseSet`` is a great way to unique small values
that are not simple pointers (use :ref:`SmallPtrSet <dss_smallptrset>` for
@@ -2416,7 +2416,7 @@ virtual register ID).
llvm/ADT/DenseMap.h
^^^^^^^^^^^^^^^^^^^
-``DenseMap`` is a simple quadratically probed hash table. It excels at supporting
+``DenseMap`` is a simple linearly probed hash table. It excels at supporting
small keys and values: it uses a single allocation to hold all of the pairs
that are currently inserted in the map. ``DenseMap`` is a great way to map
pointers to pointers, or map other small types to each other.
diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h
index b52906a215a7c..5759982d33dff 100644
--- a/llvm/include/llvm/ADT/DenseMap.h
+++ b/llvm/include/llvm/ADT/DenseMap.h
@@ -9,6 +9,13 @@
/// \file
/// This file defines the DenseMap class.
///
+/// The hash table is linear-probing open addressing with tombstone-free
+/// deletion (Knuth TAOCP 6.4 Algorithm R), power-of-two capacity, and a 0.75
+/// maximum load factor. No sentinel key. Occupancy is stored in a packed
+/// 1-bit-per-bucket "used" array.
+///
+/// `SmallDenseMap` adds an inline small buffer optimization.
+///
//===----------------------------------------------------------------------===//
#ifndef LLVM_ADT_DENSEMAP_H
@@ -28,6 +35,7 @@
#include <algorithm>
#include <cassert>
#include <cstddef>
+#include <cstdint>
#include <cstring>
#include <initializer_list>
#include <iterator>
@@ -53,6 +61,53 @@ struct DenseMapPair : std::pair<KeyT, ValueT> {
} // end namespace detail
+namespace densemap::detail {
+using used_t = uint32_t;
+
+// Number of used words backing N buckets where N is zero or a power of two.
+constexpr size_t usedWords(size_t N) {
+ assert((N == 0 || isPowerOf2_64(N)) &&
+ "bucket count must be zero or a power of two");
+ return (N + 31) / 32;
+}
+
+inline bool used(const used_t *U, size_t I) {
+ return (U[I >> 5] >> (I & 31)) & 1;
+}
+inline void setUsed(used_t *U, size_t I) { U[I >> 5] |= used_t(1) << (I & 31); }
+inline void unsetUsed(used_t *U, size_t I) {
+ U[I >> 5] &= ~(used_t(1) << (I & 31));
+}
+
+// Invoke Func(I) for each occupied bucket index I in [0, N). Set always_inline;
+// otherwise, for a heavy caller such as moveFrom's rehash, the inliner can
+// leave it out of line and the per-element call dwarfs the work.
+template <typename Fn>
+LLVM_ATTRIBUTE_ALWAYS_INLINE void forEachUsed(const used_t *U, unsigned N,
+ Fn Func) {
+ const unsigned NW = usedWords(N);
+ for (unsigned W = 0; W != NW; ++W) {
+ used_t Bits = U[W];
+ while (Bits) {
+ Func((W << 5) + llvm::countr_zero(Bits));
+ Bits &= Bits - 1;
+ }
+ }
+}
+
+// Buckets and the used array share one allocation: the bucket array first, then
+// the used words. NumBuckets is a power of two >= 4, so the bucket region size
+// is a multiple of sizeof(used_t) and the trailing used words are aligned.
+template <typename BucketT> constexpr size_t allocAlign() {
+ return std::max(alignof(BucketT), alignof(used_t));
+}
+template <typename BucketT> size_t allocBytes(unsigned Num) {
+ return sizeof(BucketT) * static_cast<size_t>(Num) +
+ usedWords(Num) * sizeof(used_t);
+}
+
+} // namespace densemap::detail
+
// Befriended below so DenseMapBase can expose its bucket-relocation callback
// erase to ValueHandleBase, the only caller that caches bucket pointers.
class ValueHandleBase;
@@ -69,6 +124,8 @@ class DenseMapBase : public DebugEpochBase {
template <typename T>
using const_arg_type_t = typename const_pointer_or_const_ref<T>::type;
+ using used_t = llvm::densemap::detail::used_t;
+
public:
using size_type = unsigned;
using key_type = KeyT;
@@ -80,16 +137,19 @@ class DenseMapBase : public DebugEpochBase {
DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT, true>;
[[nodiscard]] inline iterator begin() {
- return iterator::makeBegin(buckets(), empty(), *this);
+ return iterator::makeBegin(getBuckets(), getUsed(), getNumBuckets(),
+ empty(), *this);
}
[[nodiscard]] inline iterator end() {
- return iterator::makeEnd(buckets(), *this);
+ return iterator::makeEnd(getBuckets(), getUsed(), getNumBuckets(), *this);
}
[[nodiscard]] inline const_iterator begin() const {
- return const_iterator::makeBegin(buckets(), empty(), *this);
+ return const_iterator::makeBegin(getBuckets(), getUsed(), getNumBuckets(),
+ empty(), *this);
}
[[nodiscard]] inline const_iterator end() const {
- return const_iterator::makeEnd(buckets(), *this);
+ return const_iterator::makeEnd(getBuckets(), getUsed(), getNumBuckets(),
+ *this);
}
// Return an iterator to iterate over keys in the map.
@@ -134,23 +194,10 @@ class DenseMapBase : public DebugEpochBase {
return;
}
- const KeyT EmptyKey = KeyInfoT::getEmptyKey();
- if constexpr (std::is_trivially_destructible_v<ValueT>) {
- // Use a simpler loop when values don't need destruction.
- for (BucketT &B : buckets())
- B.getFirst() = EmptyKey;
- } else {
- unsigned NumEntries = getNumEntries();
- for (BucketT &B : buckets()) {
- if (!KeyInfoT::isEqual(B.getFirst(), EmptyKey)) {
- B.getSecond().~ValueT();
- --NumEntries;
- B.getFirst() = EmptyKey;
- }
- }
- assert(NumEntries == 0 && "Node count imbalance!");
- (void)NumEntries;
- }
+ destroyAll();
+ std::memset(getUsed(), 0,
+ llvm::densemap::detail::usedWords(getNumBuckets()) *
+ sizeof(used_t));
setNumEntries(0);
}
@@ -277,7 +324,7 @@ class DenseMapBase : public DebugEpochBase {
// Otherwise, insert the new element.
TheBucket = findBucketForInsertion(Val, TheBucket);
- TheBucket->getFirst() = std::move(KV.first);
+ ::new (&TheBucket->getFirst()) KeyT(std::move(KV.first));
::new (&TheBucket->getSecond()) ValueT(std::move(KV.second));
return {makeIterator(TheBucket), true};
}
@@ -346,15 +393,17 @@ class DenseMapBase : public DebugEpochBase {
/// Returns whether anything was removed. If so, all iterators and references
/// into the map are invalidated.
template <typename Predicate> bool remove_if(Predicate Pred) {
- const KeyT EmptyKey = KeyInfoT::getEmptyKey();
+ used_t *U = getUsed();
unsigned NumBuckets = getNumBuckets();
+ BucketT *B = getBuckets();
bool Removed = false;
- for (BucketT &B : buckets()) {
- if (KeyInfoT::isEqual(B.getFirst(), EmptyKey))
+ for (unsigned I = 0; I != NumBuckets; ++I) {
+ if (!llvm::densemap::detail::used(U, I))
continue;
- if (Pred(B)) {
- B.getSecond().~ValueT();
- B.getFirst() = EmptyKey;
+ if (Pred(B[I])) {
+ B[I].getSecond().~ValueT();
+ B[I].getFirst().~KeyT();
+ llvm::densemap::detail::unsetUsed(U, I);
decrementNumEntries();
Removed = true;
}
@@ -398,6 +447,15 @@ class DenseMapBase : public DebugEpochBase {
struct ExactBucketCount {};
+ // A snapshot of the three fields the hot lookup paths need. Fetching them
+ // together lets SmallDenseMap test its Small discriminator once rather than
+ // once per accessor; for plain DenseMap it is three member loads either way.
+ struct Rep {
+ const BucketT *Buckets;
+ const used_t *Used;
+ unsigned NumBuckets;
+ };
+
void initWithExactBucketCount(unsigned NewNumBuckets) {
if (derived().allocateBuckets(NewNumBuckets))
initEmpty();
@@ -415,12 +473,13 @@ class DenseMapBase : public DebugEpochBase {
if (getNumBuckets() == 0) // Nothing to do.
return;
- const KeyT EmptyKey = KeyInfoT::getEmptyKey();
- for (BucketT &B : buckets()) {
- if (!KeyInfoT::isEqual(B.getFirst(), EmptyKey))
- B.getSecond().~ValueT();
- B.getFirst().~KeyT();
- }
+ BucketT *B = getBuckets();
+ const used_t *U = getUsed();
+ const unsigned E = getNumBuckets();
+ llvm::densemap::detail::forEachUsed(U, E, [&](unsigned I) {
+ B[I].getSecond().~ValueT();
+ B[I].getFirst().~KeyT();
+ });
}
void initEmpty() {
@@ -430,9 +489,9 @@ class DenseMapBase : public DebugEpochBase {
assert((getNumBuckets() & (getNumBuckets() - 1)) == 0 &&
"# initial buckets must be a power of two!");
- const KeyT EmptyKey = KeyInfoT::getEmptyKey();
- for (BucketT &B : buckets())
- ::new (&B.getFirst()) KeyT(EmptyKey);
+ std::memset(getUsed(), 0,
+ llvm::densemap::detail::usedWords(getNumBuckets()) *
+ sizeof(used_t));
}
/// Returns the number of buckets to allocate to ensure that the DenseMap can
@@ -448,29 +507,34 @@ class DenseMapBase : public DebugEpochBase {
// Move key/value from Other to *this.
// Other is left in a valid but empty state.
- void moveFrom(DerivedT &Other) {
- // Insert all the old elements.
- const KeyT EmptyKey = KeyInfoT::getEmptyKey();
- for (BucketT &B : Other.buckets()) {
- if (!KeyInfoT::isEqual(B.getFirst(), EmptyKey)) {
- // Insert the key/value into the new table.
- BucketT *DestBucket;
- bool FoundVal = LookupBucketFor(B.getFirst(), DestBucket);
- (void)FoundVal; // silence warning.
- assert(!FoundVal && "Key already in new map?");
- DestBucket->getFirst() = std::move(B.getFirst());
- ::new (&DestBucket->getSecond()) ValueT(std::move(B.getSecond()));
- incrementNumEntries();
-
- // Free the value.
- B.getSecond().~ValueT();
- }
- B.getFirst().~KeyT();
- }
+ LLVM_ATTRIBUTE_NOINLINE void moveFrom(DerivedT &Other) {
+ assert(getNumEntries() == 0 && "moveFrom requires an empty destination");
+ BucketT *OtherB = Other.getBuckets();
+ used_t *OtherU = Other.getUsed();
+ const unsigned E = Other.getNumBuckets();
+ used_t *U = getUsed();
+ BucketT *B = getBuckets();
+ const unsigned Mask = getNumBuckets() - 1;
+ llvm::densemap::detail::forEachUsed(OtherU, E, [&](unsigned I) {
+ // Find the first empty slot on this key's probe chain; there is no equal
+ // key in the destination, so nothing to compare against.
+ unsigned BucketNo = KeyInfoT::getHashValue(OtherB[I].getFirst()) & Mask;
+ while (llvm::densemap::detail::used(U, BucketNo))
+ BucketNo = (BucketNo + 1) & Mask;
+ BucketT *DestBucket = B + BucketNo;
+ ::new (&DestBucket->getFirst()) KeyT(std::move(OtherB[I].getFirst()));
+ ::new (&DestBucket->getSecond()) ValueT(std::move(OtherB[I].getSecond()));
+ llvm::densemap::detail::setUsed(U, BucketNo);
+
+ // Free the moved-out key/value.
+ OtherB[I].getSecond().~ValueT();
+ OtherB[I].getFirst().~KeyT();
+ });
+ setNumEntries(Other.getNumEntries());
Other.derived().kill();
}
- void copyFrom(const DerivedT &other) {
+ LLVM_ATTRIBUTE_NOINLINE void copyFrom(const DerivedT &other) {
this->destroyAll();
derived().deallocateBuckets();
setNumEntries(0);
@@ -486,18 +550,20 @@ class DenseMapBase : public DebugEpochBase {
BucketT *Buckets = getBuckets();
const BucketT *OtherBuckets = other.getBuckets();
- const size_t NumBuckets = getNumBuckets();
+ const unsigned NumBuckets = getNumBuckets();
+ used_t *U = getUsed();
+ const used_t *OtherU = other.getUsed();
+ std::memcpy(U, OtherU,
+ llvm::densemap::detail::usedWords(NumBuckets) * sizeof(used_t));
if constexpr (std::is_trivially_copyable_v<KeyT> &&
std::is_trivially_copyable_v<ValueT>) {
memcpy(reinterpret_cast<void *>(Buckets), OtherBuckets,
NumBuckets * sizeof(BucketT));
} else {
- const KeyT EmptyKey = KeyInfoT::getEmptyKey();
- for (size_t I = 0; I < NumBuckets; ++I) {
+ llvm::densemap::detail::forEachUsed(U, NumBuckets, [&](unsigned I) {
::new (&Buckets[I].getFirst()) KeyT(OtherBuckets[I].getFirst());
- if (!KeyInfoT::isEqual(Buckets[I].getFirst(), EmptyKey))
- ::new (&Buckets[I].getSecond()) ValueT(OtherBuckets[I].getSecond());
- }
+ ::new (&Buckets[I].getSecond()) ValueT(OtherBuckets[I].getSecond());
+ });
}
}
@@ -511,35 +577,37 @@ class DenseMapBase : public DebugEpochBase {
/// TAOCP 6.4 Algorithm R. For callers that cache pointers into the bucket
/// array, call \p OnMoved per shifted bucket.
template <typename OnMovedT>
- void eraseFromFilledBucket(BucketT *TheBucket, OnMovedT &&OnMoved) {
+ LLVM_ATTRIBUTE_NOINLINE void eraseFromFilledBucket(BucketT *TheBucket,
+ OnMovedT &&OnMoved) {
incrementEpoch();
TheBucket->getSecond().~ValueT();
+ TheBucket->getFirst().~KeyT();
decrementNumEntries();
BucketT *BucketsPtr = getBuckets();
- const unsigned NumBuckets = getNumBuckets();
- const unsigned Mask = NumBuckets - 1;
- const KeyT EmptyKey = KeyInfoT::getEmptyKey();
- unsigned I = static_cast<unsigned>(TheBucket - BucketsPtr);
+ used_t *U = getUsed();
+ const unsigned Mask = getNumBuckets() - 1;
+ unsigned I = TheBucket - BucketsPtr;
unsigned J = I;
while (true) {
J = (J + 1) & Mask;
BucketT &BJ = BucketsPtr[J];
- if (KeyInfoT::isEqual(BJ.getFirst(), EmptyKey))
+ if (!llvm::densemap::detail::used(U, J))
break;
auto Ideal = KeyInfoT::getHashValue(BJ.getFirst());
// If the hole (I) lies on the linear-probe chain from the home bucket
// (Ideal) to J, shift J into the hole and make J the new hole.
if (((I - Ideal) & Mask) < ((J - Ideal) & Mask)) {
BucketT &BI = BucketsPtr[I];
- BI.getFirst() = std::move(BJ.getFirst());
+ ::new (&BI.getFirst()) KeyT(std::move(BJ.getFirst()));
::new (&BI.getSecond()) ValueT(std::move(BJ.getSecond()));
BJ.getSecond().~ValueT();
+ BJ.getFirst().~KeyT();
OnMoved(BI);
I = J;
}
}
- BucketsPtr[I].getFirst() = EmptyKey;
+ llvm::densemap::detail::unsetUsed(U, I);
}
/// Erase \p Val and close the resulting hole by potentially shifting other
@@ -567,7 +635,7 @@ class DenseMapBase : public DebugEpochBase {
// Otherwise, insert the new element.
TheBucket = findBucketForInsertion(Key, TheBucket);
- TheBucket->getFirst() = std::forward<KeyArgT>(Key);
+ ::new (&TheBucket->getFirst()) KeyT(std::forward<KeyArgT>(Key));
::new (&TheBucket->getSecond()) ValueT(std::forward<Ts>(Args)...);
return {TheBucket, true};
}
@@ -580,11 +648,13 @@ class DenseMapBase : public DebugEpochBase {
}
iterator makeIterator(BucketT *TheBucket) {
- return iterator::makeIterator(TheBucket, buckets(), *this);
+ return iterator::makeIterator(TheBucket, getBuckets(), getUsed(),
+ getNumBuckets(), *this);
}
const_iterator makeConstIterator(const BucketT *TheBucket) const {
- return const_iterator::makeIterator(TheBucket, buckets(), *this);
+ return const_iterator::makeIterator(TheBucket, getBuckets(), getUsed(),
+ getNumBuckets(), *this);
}
unsigned getNumEntries() const { return derived().getNumEntries(); }
@@ -599,6 +669,12 @@ class DenseMapBase : public DebugEpochBase {
BucketT *getBuckets() { return derived().getBuckets(); }
+ Rep getRep() const { return derived().getRep(); }
+
+ const used_t *getUsed() const { return derived().getUsed(); }
+
+ used_t *getUsed() { return derived().getUsed(); }
+
unsigned getNumBuckets() const { return derived().getNumBuckets(); }
BucketT *getBucketsEnd() { return getBuckets() + getNumBuckets(); }
@@ -607,15 +683,7 @@ class DenseMapBase : public DebugEpochBase {
return getBuckets() + getNumBuckets();
}
- iterator_range<BucketT *> buckets() {
- return llvm::make_range(getBuckets(), getBucketsEnd());
- }
-
- iterator_range<const BucketT *> buckets() const {
- return llvm::make_range(getBuckets(), getBucketsEnd());
- }
-
- void grow(unsigned MinNumBuckets) {
+ LLVM_ATTRIBUTE_NOINLINE void grow(unsigned MinNumBuckets) {
unsigned NumBuckets = DerivedT::roundUpNumBuckets(MinNumBuckets);
DerivedT Tmp(NumBuckets, ExactBucketCount{});
Tmp.moveFrom(derived());
@@ -642,6 +710,9 @@ class DenseMapBase : public DebugEpochBase {
}
assert(TheBucket);
+ // Mark used. The caller will placement-construct the raw key/value.
+ llvm::densemap::detail::setUsed(getUsed(), TheBucket - getBuckets());
+
// Only update the state after we've grown our bucket space appropriately
// so that when growing buckets we have self-consistent entry count.
incrementNumEntries();
@@ -650,22 +721,22 @@ class DenseMapBase : public DebugEpochBase {
template <typename LookupKeyT>
const BucketT *doFind(const LookupKeyT &Val) const {
- const BucketT *BucketsPtr = getBuckets();
- const unsigned NumBuckets = getNumBuckets();
+ auto [BucketsPtr, U, NumBuckets] = getRep();
if (NumBuckets == 0)
return nullptr;
- const KeyT EmptyKey = KeyInfoT::getEmptyKey();
- unsigned BucketNo = KeyInfoT::getHashValue(Val) & (NumBuckets - 1);
+ const unsigned Mask = NumBuckets - 1;
+ unsigned BucketNo = KeyInfoT::getHashValue(Val) & Mask;
while (true) {
+ // An empty bucket terminates the probe: the key isn't in the map.
+ if (LLVM_LIKELY(!llvm::densemap::detail::used(U, BucketNo)))
+ return nullptr;
const BucketT *Bucket = BucketsPtr + BucketNo;
if (LLVM_LIKELY(KeyInfoT::isEqual(Val, Bucket->getFirst())))
return Bucket;
- if (LLVM_LIKELY(KeyInfoT::isEqual(Bucket->getFirst(), EmptyKey)))
- return nullptr;
// Hash collision: continue linear probing.
- BucketNo = (BucketNo + 1) & (NumBuckets - 1);
+ BucketNo = (BucketNo + 1) & Mask;
}
}
@@ -679,36 +750,34 @@ class DenseMapBase : public DebugEpochBase {
/// returns a bucket with an empty marker and returns false.
template <typename LookupKeyT>
bool LookupBucketFor(const LookupKeyT &Val, BucketT *&FoundBucket) {
- BucketT *BucketsPtr = getBuckets();
- const unsigned NumBuckets = getNumBuckets();
-
+ auto [CBuckets, U, NumBuckets] = getRep();
if (NumBuckets == 0) {
FoundBucket = nullptr;
return false;
}
+ // getRep() yields const pointers; this object is non-const, so recovering
+ // a mutable bucket pointer is safe (mirrors the non-const getBuckets()).
+ BucketT *BucketsPtr = const_cast<BucketT *>(CBuckets);
- const KeyT EmptyKey = KeyInfoT::getEmptyKey();
- assert(!KeyInfoT::isEqual(Val, EmptyKey) &&
- "Empty value shouldn't be inserted into map!");
-
- unsigned BucketNo = KeyInfoT::getHashValue(Val) & (NumBuckets - 1);
+ const unsigned Mask = NumBuckets - 1;
+ unsigned BucketNo = KeyInfoT::getHashValue(Val) & Mask;
while (true) {
BucketT *ThisBucket = BucketsPtr + BucketNo;
- // Found Val's bucket? If so, return it.
- if (LLVM_LIKELY(KeyInfoT::isEqual(Val, ThisBucket->getFirst()))) {
- FoundBucket = ThisBucket;
- return true;
- }
-
// If we found an empty bucket, the key doesn't exist in the set.
// Return it as the insertion point.
- if (LLVM_LIKELY(KeyInfoT::isEqual(ThisBucket->getFirst(), EmptyKey))) {
+ if (LLVM_LIKELY(!llvm::densemap::detail::used(U, BucketNo))) {
FoundBucket = ThisBucket;
return false;
}
+ // Found Val's bucket? If so, return it.
+ if (LLVM_LIKELY(KeyInfoT::isEqual(Val, ThisBucket->getFirst()))) {
+ FoundBucket = ThisBucket;
+ return true;
+ }
+
// Hash collision: continue linear probing.
- BucketNo = (BucketNo + 1) & (NumBuckets - 1);
+ BucketNo = (BucketNo + 1) & Mask;
}
}
@@ -718,7 +787,7 @@ class DenseMapBase : public DebugEpochBase {
/// If entries are pointers to objects, the size of the referenced objects
/// are not included.
[[nodiscard]] size_t getMemorySize() const {
- return getNumBuckets() * sizeof(BucketT);
+ return llvm::densemap::detail::allocBytes<BucketT>(getNumBuckets());
}
};
@@ -766,8 +835,10 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
// Lift some types from the dependent base class into this class for
// simplicity of referring to them.
using BaseT = DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
+ using used_t = llvm::densemap::detail::used_t;
BucketT *Buckets = nullptr;
+ used_t *Used = nullptr;
unsigned NumEntries = 0;
unsigned NumBuckets = 0;
@@ -820,6 +891,7 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
private:
void swapImpl(DenseMap &RHS) {
std::swap(Buckets, RHS.Buckets);
+ std::swap(Used, RHS.Used);
std::swap(NumEntries, RHS.NumEntries);
std::swap(NumBuckets, RHS.NumBuckets);
}
@@ -830,30 +902,42 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
BucketT *getBuckets() const { return Buckets; }
+ typename BaseT::Rep getRep() const { return {Buckets, Used, NumBuckets}; }
+
+ used_t *getUsed() const { return Used; }
+
unsigned getNumBuckets() const { return NumBuckets; }
void deallocateBuckets() {
- deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT));
+ if (NumBuckets == 0)
+ return;
+ deallocate_buffer(Buckets,
+ llvm::densemap::detail::allocBytes<BucketT>(NumBuckets),
+ llvm::densemap::detail::allocAlign<BucketT>());
+ Buckets = nullptr;
+ Used = nullptr;
+ NumBuckets = 0;
}
bool allocateBuckets(unsigned Num) {
NumBuckets = Num;
if (NumBuckets == 0) {
Buckets = nullptr;
+ Used = nullptr;
return false;
}
- Buckets = static_cast<BucketT *>(
- allocate_buffer(sizeof(BucketT) * NumBuckets, alignof(BucketT)));
+ auto *Storage = static_cast<char *>(
+ allocate_buffer(llvm::densemap::detail::allocBytes<BucketT>(NumBuckets),
+ llvm::densemap::detail::allocAlign<BucketT>()));
+ Buckets = reinterpret_cast<BucketT *>(Storage);
+ Used = reinterpret_cast<used_t *>(Storage + sizeof(BucketT) * NumBuckets);
return true;
}
// Put the zombie instance in a known good state after a move.
- void kill() {
- deallocateBuckets();
- Buckets = nullptr;
- NumBuckets = 0;
- }
+ // deallocateBuckets() already resets to the empty state.
+ void kill() { deallocateBuckets(); }
static unsigned roundUpNumBuckets(unsigned MinNumBuckets) {
return std::max(64u,
@@ -890,24 +974,34 @@ class SmallDenseMap
// Lift some types from the dependent base class into this class for
// simplicity of referring to them.
using BaseT = DenseMapBase<SmallDenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
+ using used_t = llvm::densemap::detail::used_t;
static_assert(isPowerOf2_64(InlineBuckets),
"InlineBuckets must be a power of 2.");
+ // Number of used words backing the inline buckets (>= 1).
+ static constexpr unsigned InlineUsedWords =
+ llvm::densemap::detail::usedWords(InlineBuckets);
+
unsigned Small : 1;
unsigned NumEntries : 31;
+ // Inline storage: the bucket array followed by the parallel used words.
+ struct InlineRep {
+ alignas(BucketT) char Buckets[sizeof(BucketT) * InlineBuckets];
+ used_t Used[InlineUsedWords];
+ };
struct LargeRep {
BucketT *Buckets;
+ used_t *Used;
unsigned NumBuckets;
- iterator_range<BucketT *> buckets() {
- return llvm::make_range(Buckets, Buckets + NumBuckets);
- }
};
- /// A "union" of an inline bucket array and the struct representing
- /// a large bucket. This union will be discriminated by the 'Small' bit.
- AlignedCharArrayUnion<BucketT[InlineBuckets], LargeRep> storage;
+ // Discriminated by the Small bit.
+ union {
+ InlineRep Inline;
+ LargeRep Large;
+ } storage;
SmallDenseMap(unsigned NumBuckets, typename BaseT::ExactBucketCount) {
this->initWithExactBucketCount(NumBuckets);
@@ -958,70 +1052,70 @@ class SmallDenseMap
}
private:
+ // Move-construct *Dst from *Src, then destroy *Src. Dst is raw storage.
+ static void relocateBucket(BucketT *Dst, BucketT *Src) {
+ ::new (&Dst->getFirst()) KeyT(std::move(Src->getFirst()));
+ ::new (&Dst->getSecond()) ValueT(std::move(Src->getSecond()));
+ Src->getSecond().~ValueT();
+ Src->getFirst().~KeyT();
+ }
+
void swapImpl(SmallDenseMap &RHS) {
unsigned TmpNumEntries = RHS.NumEntries;
RHS.NumEntries = NumEntries;
NumEntries = TmpNumEntries;
- const KeyT EmptyKey = KeyInfoT::getEmptyKey();
if (Small && RHS.Small) {
- // If we're swapping inline bucket arrays, we have to cope with some of
- // the tricky bits of DenseMap's storage system: the buckets are not
- // fully initialized. Thus we swap every key, but we may have
- // a one-directional move of the value.
- for (unsigned i = 0, e = InlineBuckets; i != e; ++i) {
- BucketT *LHSB = &getInlineBuckets()[i],
- *RHSB = &RHS.getInlineBuckets()[i];
- bool hasLHSValue = !KeyInfoT::isEqual(LHSB->getFirst(), EmptyKey);
- bool hasRHSValue = !KeyInfoT::isEqual(RHSB->getFirst(), EmptyKey);
- if (hasLHSValue && hasRHSValue) {
- // Swap together if we can...
- std::swap(*LHSB, *RHSB);
- continue;
- }
- // Swap separately and handle any asymmetry.
- std::swap(LHSB->getFirst(), RHSB->getFirst());
- if (hasLHSValue) {
- ::new (&RHSB->getSecond()) ValueT(std::move(LHSB->getSecond()));
- LHSB->getSecond().~ValueT();
- } else if (hasRHSValue) {
- ::new (&LHSB->getSecond()) ValueT(std::move(RHSB->getSecond()));
- RHSB->getSecond().~ValueT();
+ // Both inline: swap the live bucket contents slot by slot, then the used
+ // used words. Buckets are raw storage, so a value may only move in one
+ // direction when exactly one side is occupied.
+ used_t *LU = getInlineUsed(), *RU = RHS.getInlineUsed();
+ BucketT *LB = getInlineBuckets(), *RB = RHS.getInlineBuckets();
+ for (unsigned I = 0; I != InlineBuckets; ++I) {
+ bool L = llvm::densemap::detail::used(LU, I);
+ bool R = llvm::densemap::detail::used(RU, I);
+ if (L && R) {
+ // Both occupied: exchange through a temporary.
+ alignas(BucketT) char Tmp[sizeof(BucketT)];
+ BucketT *T = reinterpret_cast<BucketT *>(Tmp);
+ relocateBucket(T, &LB[I]);
+ relocateBucket(&LB[I], &RB[I]);
+ relocateBucket(&RB[I], T);
+ } else if (L) {
+ relocateBucket(&RB[I], &LB[I]);
+ } else if (R) {
+ relocateBucket(&LB[I], &RB[I]);
}
}
+ for (unsigned W = 0; W != InlineUsedWords; ++W)
+ std::swap(LU[W], RU[W]);
return;
}
if (!Small && !RHS.Small) {
- std::swap(*getLargeRep(), *RHS.getLargeRep());
+ std::swap(storage.Large, RHS.storage.Large);
return;
}
SmallDenseMap &SmallSide = Small ? *this : RHS;
SmallDenseMap &LargeSide = Small ? RHS : *this;
- // First stash the large side's rep and move the small side across.
- LargeRep TmpRep = std::move(*LargeSide.getLargeRep());
- LargeSide.getLargeRep()->~LargeRep();
+ // Stash the large rep, then move the small side's inline contents into the
+ // large side (which becomes inline), and finally install the rep on the
+ // small side (which becomes large).
+ LargeRep TmpRep = LargeSide.storage.Large;
LargeSide.Small = true;
- // This is similar to the standard move-from-old-buckets, but the bucket
- // count hasn't actually rotated in this case. So we have to carefully
- // move construct the keys and values into their new locations, but there
- // is no need to re-hash things.
- for (unsigned i = 0, e = InlineBuckets; i != e; ++i) {
- BucketT *NewB = &LargeSide.getInlineBuckets()[i],
- *OldB = &SmallSide.getInlineBuckets()[i];
- ::new (&NewB->getFirst()) KeyT(std::move(OldB->getFirst()));
- OldB->getFirst().~KeyT();
- if (!KeyInfoT::isEqual(NewB->getFirst(), EmptyKey)) {
- ::new (&NewB->getSecond()) ValueT(std::move(OldB->getSecond()));
- OldB->getSecond().~ValueT();
- }
+ {
+ used_t *SU = SmallSide.getInlineUsed(), *LU = LargeSide.getInlineUsed();
+ BucketT *SB = SmallSide.getInlineBuckets(),
+ *LB = LargeSide.getInlineBuckets();
+ for (unsigned I = 0; I != InlineBuckets; ++I)
+ if (llvm::densemap::detail::used(SU, I))
+ relocateBucket(&LB[I], &SB[I]);
+ for (unsigned W = 0; W != InlineUsedWords; ++W)
+ LU[W] = SU[W];
}
-
- // The hard part of moving the small buckets across is done, just move
- // the TmpRep into its new home.
SmallSide.Small = false;
- new (SmallSide.getLargeRep()) LargeRep(std::move(TmpRep));
+ SmallSide.storage.Large = TmpRep;
}
unsigned getNumEntries() const { return NumEntries; }
@@ -1037,27 +1131,33 @@ class SmallDenseMap
// Note that this cast does not violate aliasing rules as we assert that
// the memory's dynamic type is the small, inline bucket buffer, and the
// 'storage' is a POD containing a char buffer.
- return reinterpret_cast<const BucketT *>(&storage);
+ return reinterpret_cast<const BucketT *>(storage.Inline.Buckets);
}
BucketT *getInlineBuckets() {
- return const_cast<BucketT *>(
- const_cast<const SmallDenseMap *>(this)->getInlineBuckets());
+ assert(Small);
+ return reinterpret_cast<BucketT *>(storage.Inline.Buckets);
}
- const LargeRep *getLargeRep() const {
- assert(!Small);
- // Note, same rule about aliasing as with getInlineBuckets.
- return reinterpret_cast<const LargeRep *>(&storage);
+ const used_t *getInlineUsed() const {
+ assert(Small);
+ return storage.Inline.Used;
}
- LargeRep *getLargeRep() {
- return const_cast<LargeRep *>(
- const_cast<const SmallDenseMap *>(this)->getLargeRep());
+ used_t *getInlineUsed() {
+ assert(Small);
+ return storage.Inline.Used;
}
const BucketT *getBuckets() const {
- return Small ? getInlineBuckets() : getLargeRep()->Buckets;
+ return Small ? getInlineBuckets() : storage.Large.Buckets;
+ }
+
+ typename BaseT::Rep getRep() const {
+ if (Small)
+ return {getInlineBuckets(), getInlineUsed(), InlineBuckets};
+ return {storage.Large.Buckets, storage.Large.Used,
+ storage.Large.NumBuckets};
}
BucketT *getBuckets() {
@@ -1065,37 +1165,44 @@ class SmallDenseMap
const_cast<const SmallDenseMap *>(this)->getBuckets());
}
- unsigned getNumBuckets() const {
- return Small ? InlineBuckets : getLargeRep()->NumBuckets;
+ const used_t *getUsed() const {
+ return Small ? getInlineUsed() : storage.Large.Used;
}
- iterator_range<BucketT *> inlineBuckets() {
- BucketT *Begin = getInlineBuckets();
- return llvm::make_range(Begin, Begin + InlineBuckets);
+ used_t *getUsed() {
+ return const_cast<used_t *>(
+ const_cast<const SmallDenseMap *>(this)->getUsed());
+ }
+
+ unsigned getNumBuckets() const {
+ return Small ? InlineBuckets : storage.Large.NumBuckets;
}
void deallocateBuckets() {
- // Fast path to deallocateBuckets in case getLargeRep()->NumBuckets == 0,
- // just like destroyAll. This path is used to destruct zombie instances
- // after moves.
- if (Small || getLargeRep()->NumBuckets == 0)
+ // Fast path in case storage.Large.NumBuckets == 0, just like destroyAll.
+ // This path is used to destruct zombie instances after moves.
+ if (Small || storage.Large.NumBuckets == 0)
return;
- deallocate_buffer(getLargeRep()->Buckets,
- sizeof(BucketT) * getLargeRep()->NumBuckets,
- alignof(BucketT));
- getLargeRep()->~LargeRep();
+ deallocate_buffer(
+ storage.Large.Buckets,
+ llvm::densemap::detail::allocBytes<BucketT>(storage.Large.NumBuckets),
+ llvm::densemap::detail::allocAlign<BucketT>());
+ storage.Large.NumBuckets = 0;
}
bool allocateBuckets(unsigned Num) {
if (Num <= InlineBuckets) {
Small = true;
- } else {
- Small = false;
- BucketT *NewBuckets = static_cast<BucketT *>(
- allocate_buffer(sizeof(BucketT) * Num, alignof(BucketT)));
- new (getLargeRep()) LargeRep{NewBuckets, Num};
+ return true;
}
+ Small = false;
+ auto *S = static_cast<char *>(
+ allocate_buffer(llvm::densemap::detail::allocBytes<BucketT>(Num),
+ llvm::densemap::detail::allocAlign<BucketT>()));
+ storage.Large.Buckets = reinterpret_cast<BucketT *>(S);
+ storage.Large.Used = reinterpret_cast<used_t *>(S + sizeof(BucketT) * Num);
+ storage.Large.NumBuckets = Num;
return true;
}
@@ -1103,12 +1210,12 @@ class SmallDenseMap
void kill() {
deallocateBuckets();
Small = false;
- new (getLargeRep()) LargeRep{nullptr, 0};
+ storage.Large = LargeRep{nullptr, nullptr, 0};
}
static unsigned roundUpNumBuckets(unsigned MinNumBuckets) {
if (MinNumBuckets <= InlineBuckets)
- return MinNumBuckets;
+ return InlineBuckets;
return std::max(64u,
static_cast<unsigned>(NextPowerOf2(MinNumBuckets - 1)));
}
@@ -1119,8 +1226,8 @@ class SmallDenseMap
Small = false;
NumEntries = Other.NumEntries;
- *getLargeRep() = std::move(*Other.getLargeRep());
- Other.getLargeRep()->NumBuckets = 0;
+ storage.Large = Other.storage.Large;
+ Other.storage.Large.NumBuckets = 0;
return true;
}
@@ -1135,7 +1242,7 @@ class SmallDenseMap
NewNumBuckets = std::max(64u, NewNumBuckets);
}
bool Reuse = Small ? NewNumBuckets <= InlineBuckets
- : NewNumBuckets == getLargeRep()->NumBuckets;
+ : NewNumBuckets == storage.Large.NumBuckets;
if (Reuse)
return {false, 0}; // Reuse.
return {true, NewNumBuckets}; // Reallocate.
@@ -1148,6 +1255,8 @@ class DenseMapIterator : DebugEpochBase::HandleBase {
friend class DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, true>;
friend class DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, false>;
+ using used_t = llvm::densemap::detail::used_t;
+
public:
using difference_type = ptrdiff_t;
using value_type = std::conditional_t<IsConst, const Bucket, Bucket>;
@@ -1162,39 +1271,48 @@ class DenseMapIterator : DebugEpochBase::HandleBase {
BucketItTy Ptr = {};
BucketItTy End = {};
-
- DenseMapIterator(BucketItTy Pos, BucketItTy E, const DebugEpochBase &Epoch)
- : DebugEpochBase::HandleBase(&Epoch), Ptr(Pos), End(E) {
+ // The non-reversed bucket base and the parallel used array. They map a
+ // bucket back to its index so AdvancePastEmptyBuckets can consult the bits.
+ pointer Buckets = {};
+ const used_t *Used = {};
+
+ DenseMapIterator(BucketItTy Pos, BucketItTy E, pointer BucketsBase,
+ const used_t *U, const DebugEpochBase &Epoch)
+ : DebugEpochBase::HandleBase(&Epoch), Ptr(Pos), End(E),
+ Buckets(BucketsBase), Used(U) {
assert(isHandleInSync() && "invalid construction!");
}
public:
DenseMapIterator() = default;
- static DenseMapIterator makeBegin(iterator_range<pointer> Buckets,
- bool IsEmpty, const DebugEpochBase &Epoch) {
+ static DenseMapIterator makeBegin(pointer Buckets, const used_t *Used,
+ unsigned NumBuckets, bool IsEmpty,
+ const DebugEpochBase &Epoch) {
// When the map is empty, avoid the overhead of advancing/retreating past
// empty buckets.
if (IsEmpty)
- return makeEnd(Buckets, Epoch);
- auto R = maybeReverse(Buckets);
- DenseMapIterator Iter(R.begin(), R.end(), Epoch);
+ return makeEnd(Buckets, Used, NumBuckets, Epoch);
+ auto R = maybeReverse(llvm::make_range(Buckets, Buckets + NumBuckets));
+ DenseMapIterator Iter(R.begin(), R.end(), Buckets, Used, Epoch);
Iter.AdvancePastEmptyBuckets();
return Iter;
}
- static DenseMapIterator makeEnd(iterator_range<pointer> Buckets,
+ static DenseMapIterator makeEnd(pointer Buckets, const used_t *Used,
+ unsigned NumBuckets,
const DebugEpochBase &Epoch) {
- auto R = maybeReverse(Buckets);
- return DenseMapIterator(R.end(), R.end(), Epoch);
+ auto R = maybeReverse(llvm::make_range(Buckets, Buckets + NumBuckets));
+ return DenseMapIterator(R.end(), R.end(), Buckets, Used, Epoch);
}
- static DenseMapIterator makeIterator(pointer P,
- iterator_range<pointer> Buckets,
+ static DenseMapIterator makeIterator(pointer P, pointer Buckets,
+ const used_t *Used, unsigned NumBuckets,
const DebugEpochBase &Epoch) {
- auto R = maybeReverse(Buckets);
+ auto R = maybeReverse(llvm::make_range(Buckets, Buckets + NumBuckets));
constexpr int Offset = shouldReverseIterate<KeyT>() ? 1 : 0;
- return DenseMapIterator(BucketItTy(P + Offset), R.end(), Epoch);
+ return DenseMapIterator(BucketItTy(P + Offset), R.end(), Buckets, Used,
+ Epoch);
}
// Converting ctor from non-const iterators to const iterators. SFINAE'd out
@@ -1204,7 +1322,8 @@ class DenseMapIterator : DebugEpochBase::HandleBase {
typename = std::enable_if_t<!IsConstSrc && IsConst>>
DenseMapIterator(
const DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, IsConstSrc> &I)
- : DebugEpochBase::HandleBase(I), Ptr(I.Ptr), End(I.End) {}
+ : DebugEpochBase::HandleBase(I), Ptr(I.Ptr), End(I.End),
+ Buckets(I.Buckets), Used(I.Used) {}
[[nodiscard]] reference operator*() const {
assert(isHandleInSync() && "invalid iterator access!");
@@ -1245,11 +1364,30 @@ class DenseMapIterator : DebugEpochBase::HandleBase {
private:
void AdvancePastEmptyBuckets() {
- assert(Ptr <= End);
- const KeyT Empty = KeyInfoT::getEmptyKey();
-
- while (Ptr != End && KeyInfoT::isEqual(Ptr->getFirst(), Empty))
- ++Ptr;
+ if constexpr (shouldReverseIterate<KeyT>()) {
+ while (Ptr != End && !llvm::densemap::detail::used(Used, &*Ptr - Buckets))
+ ++Ptr;
+ } else {
+ // Forward iteration skips empty buckets a used-word (32 buckets) at a
+ // time: scan from the current index for the next set occupancy bit.
+ const size_t N = End - Buckets;
+ size_t I = Ptr - Buckets;
+ if (I >= N) {
+ Ptr = End;
+ return;
+ }
+ const size_t NW = llvm::densemap::detail::usedWords(N);
+ size_t W = I >> 5;
+ used_t Bits = Used[W] & (~used_t(0) << (I & 31));
+ while (Bits == 0) {
+ if (++W == NW) {
+ Ptr = End;
+ return;
+ }
+ Bits = Used[W];
+ }
+ Ptr = Buckets + ((W << 5) + llvm::countr_zero(Bits));
+ }
}
static auto maybeReverse(iterator_range<pointer> Range) {
diff --git a/llvm/unittests/ADT/BitVectorTest.cpp b/llvm/unittests/ADT/BitVectorTest.cpp
index adf788b8f9661..14f14660048b3 100644
--- a/llvm/unittests/ADT/BitVectorTest.cpp
+++ b/llvm/unittests/ADT/BitVectorTest.cpp
@@ -1432,10 +1432,13 @@ TYPED_TEST(BitVectorTest, DenseSet) {
I = Set.insert(C);
EXPECT_EQ(true, I.second);
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+ // Occupancy lives in the used-bit array rather than a sentinel key stored in
+ // a bucket, so a value equal to the Empty/Tombstone key can be inserted like
+ // any other.
TypeParam D;
- EXPECT_DEATH(Set.insert(D), "Empty value shouldn't be inserted into map!");
-#endif
+ EXPECT_EQ(true, Set.insert(D).second);
+ EXPECT_EQ(1U, Set.count(D));
+ EXPECT_EQ(true, Set.erase(D));
EXPECT_EQ(3U, Set.size());
EXPECT_EQ(1U, Set.count(A));
diff --git a/llvm/unittests/ADT/DenseMapTest.cpp b/llvm/unittests/ADT/DenseMapTest.cpp
index e191577249f74..88c3fd112cc98 100644
--- a/llvm/unittests/ADT/DenseMapTest.cpp
+++ b/llvm/unittests/ADT/DenseMapTest.cpp
@@ -976,6 +976,9 @@ TEST(DenseMapCustomTest, PairPrinting) {
TEST(DenseMapCustomTest, InitSize) {
constexpr unsigned ElemSize = sizeof(std::pair<int *, int>);
+ // getMemorySize() counts the bucket array plus the packed used-bit array: one
+ // uint32 word per 32 buckets (a single word covers all counts used here).
+ constexpr unsigned UsedSize = sizeof(uint32_t);
{
DenseMap<int *, int> Map;
@@ -987,59 +990,62 @@ TEST(DenseMapCustomTest, InitSize) {
}
{
DenseMap<int *, int> Map(1);
- EXPECT_EQ(ElemSize * 4U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 4U + UsedSize, Map.getMemorySize());
}
{
DenseMap<int *, int> Map(2);
- EXPECT_EQ(ElemSize * 4U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 4U + UsedSize, Map.getMemorySize());
}
{
DenseMap<int *, int> Map(3);
- EXPECT_EQ(ElemSize * 8U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 8U + UsedSize, Map.getMemorySize());
}
{
int A, B;
DenseMap<int *, int> Map = {{&A, 1}, {&B, 2}};
- EXPECT_EQ(ElemSize * 4U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 4U + UsedSize, Map.getMemorySize());
}
{
int A, B, C;
DenseMap<int *, int> Map = {{&A, 1}, {&B, 2}, {&C, 3}};
- EXPECT_EQ(ElemSize * 8U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 8U + UsedSize, Map.getMemorySize());
}
}
TEST(SmallDenseMapCustomTest, InitSize) {
constexpr unsigned ElemSize = sizeof(std::pair<int *, int>);
+ // getMemorySize() counts the bucket array plus the packed used-bit array: one
+ // uint32 word per 32 buckets (a single word covers all counts used here).
+ constexpr unsigned UsedSize = sizeof(uint32_t);
{
SmallDenseMap<int *, int> Map;
- EXPECT_EQ(ElemSize * 4U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 4U + UsedSize, Map.getMemorySize());
}
{
SmallDenseMap<int *, int> Map(0);
- EXPECT_EQ(ElemSize * 4U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 4U + UsedSize, Map.getMemorySize());
}
{
SmallDenseMap<int *, int> Map(1);
- EXPECT_EQ(ElemSize * 4U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 4U + UsedSize, Map.getMemorySize());
}
{
SmallDenseMap<int *, int> Map(2);
- EXPECT_EQ(ElemSize * 4U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 4U + UsedSize, Map.getMemorySize());
}
{
SmallDenseMap<int *, int> Map(3);
- EXPECT_EQ(ElemSize * 8U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 8U + UsedSize, Map.getMemorySize());
}
{
int A, B;
SmallDenseMap<int *, int> Map = {{&A, 1}, {&B, 2}};
- EXPECT_EQ(ElemSize * 4U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 4U + UsedSize, Map.getMemorySize());
}
{
int A, B, C;
SmallDenseMap<int *, int> Map = {{&A, 1}, {&B, 2}, {&C, 3}};
- EXPECT_EQ(ElemSize * 8U, Map.getMemorySize());
+ EXPECT_EQ(ElemSize * 8U + UsedSize, Map.getMemorySize());
}
}
>From 340bc643aa9921327648a5d275821b642d4bc643 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Wed, 3 Jun 2026 06:51:48 -0700
Subject: [PATCH 2/2] rename to UsedT
---
llvm/include/llvm/ADT/DenseMap.h | 92 ++++++++++++++++----------------
1 file changed, 46 insertions(+), 46 deletions(-)
diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h
index 5759982d33dff..54294d15841f4 100644
--- a/llvm/include/llvm/ADT/DenseMap.h
+++ b/llvm/include/llvm/ADT/DenseMap.h
@@ -62,7 +62,7 @@ struct DenseMapPair : std::pair<KeyT, ValueT> {
} // end namespace detail
namespace densemap::detail {
-using used_t = uint32_t;
+using UsedT = uint32_t;
// Number of used words backing N buckets where N is zero or a power of two.
constexpr size_t usedWords(size_t N) {
@@ -71,23 +71,23 @@ constexpr size_t usedWords(size_t N) {
return (N + 31) / 32;
}
-inline bool used(const used_t *U, size_t I) {
+inline bool used(const UsedT *U, size_t I) {
return (U[I >> 5] >> (I & 31)) & 1;
}
-inline void setUsed(used_t *U, size_t I) { U[I >> 5] |= used_t(1) << (I & 31); }
-inline void unsetUsed(used_t *U, size_t I) {
- U[I >> 5] &= ~(used_t(1) << (I & 31));
+inline void setUsed(UsedT *U, size_t I) { U[I >> 5] |= UsedT(1) << (I & 31); }
+inline void unsetUsed(UsedT *U, size_t I) {
+ U[I >> 5] &= ~(UsedT(1) << (I & 31));
}
// Invoke Func(I) for each occupied bucket index I in [0, N). Set always_inline;
// otherwise, for a heavy caller such as moveFrom's rehash, the inliner can
// leave it out of line and the per-element call dwarfs the work.
template <typename Fn>
-LLVM_ATTRIBUTE_ALWAYS_INLINE void forEachUsed(const used_t *U, unsigned N,
+LLVM_ATTRIBUTE_ALWAYS_INLINE void forEachUsed(const UsedT *U, unsigned N,
Fn Func) {
const unsigned NW = usedWords(N);
for (unsigned W = 0; W != NW; ++W) {
- used_t Bits = U[W];
+ UsedT Bits = U[W];
while (Bits) {
Func((W << 5) + llvm::countr_zero(Bits));
Bits &= Bits - 1;
@@ -97,13 +97,13 @@ LLVM_ATTRIBUTE_ALWAYS_INLINE void forEachUsed(const used_t *U, unsigned N,
// Buckets and the used array share one allocation: the bucket array first, then
// the used words. NumBuckets is a power of two >= 4, so the bucket region size
-// is a multiple of sizeof(used_t) and the trailing used words are aligned.
+// is a multiple of sizeof(UsedT) and the trailing used words are aligned.
template <typename BucketT> constexpr size_t allocAlign() {
- return std::max(alignof(BucketT), alignof(used_t));
+ return std::max(alignof(BucketT), alignof(UsedT));
}
template <typename BucketT> size_t allocBytes(unsigned Num) {
return sizeof(BucketT) * static_cast<size_t>(Num) +
- usedWords(Num) * sizeof(used_t);
+ usedWords(Num) * sizeof(UsedT);
}
} // namespace densemap::detail
@@ -124,7 +124,7 @@ class DenseMapBase : public DebugEpochBase {
template <typename T>
using const_arg_type_t = typename const_pointer_or_const_ref<T>::type;
- using used_t = llvm::densemap::detail::used_t;
+ using UsedT = llvm::densemap::detail::UsedT;
public:
using size_type = unsigned;
@@ -197,7 +197,7 @@ class DenseMapBase : public DebugEpochBase {
destroyAll();
std::memset(getUsed(), 0,
llvm::densemap::detail::usedWords(getNumBuckets()) *
- sizeof(used_t));
+ sizeof(UsedT));
setNumEntries(0);
}
@@ -393,7 +393,7 @@ class DenseMapBase : public DebugEpochBase {
/// Returns whether anything was removed. If so, all iterators and references
/// into the map are invalidated.
template <typename Predicate> bool remove_if(Predicate Pred) {
- used_t *U = getUsed();
+ UsedT *U = getUsed();
unsigned NumBuckets = getNumBuckets();
BucketT *B = getBuckets();
bool Removed = false;
@@ -452,7 +452,7 @@ class DenseMapBase : public DebugEpochBase {
// once per accessor; for plain DenseMap it is three member loads either way.
struct Rep {
const BucketT *Buckets;
- const used_t *Used;
+ const UsedT *Used;
unsigned NumBuckets;
};
@@ -474,7 +474,7 @@ class DenseMapBase : public DebugEpochBase {
return;
BucketT *B = getBuckets();
- const used_t *U = getUsed();
+ const UsedT *U = getUsed();
const unsigned E = getNumBuckets();
llvm::densemap::detail::forEachUsed(U, E, [&](unsigned I) {
B[I].getSecond().~ValueT();
@@ -491,7 +491,7 @@ class DenseMapBase : public DebugEpochBase {
"# initial buckets must be a power of two!");
std::memset(getUsed(), 0,
llvm::densemap::detail::usedWords(getNumBuckets()) *
- sizeof(used_t));
+ sizeof(UsedT));
}
/// Returns the number of buckets to allocate to ensure that the DenseMap can
@@ -510,9 +510,9 @@ class DenseMapBase : public DebugEpochBase {
LLVM_ATTRIBUTE_NOINLINE void moveFrom(DerivedT &Other) {
assert(getNumEntries() == 0 && "moveFrom requires an empty destination");
BucketT *OtherB = Other.getBuckets();
- used_t *OtherU = Other.getUsed();
+ UsedT *OtherU = Other.getUsed();
const unsigned E = Other.getNumBuckets();
- used_t *U = getUsed();
+ UsedT *U = getUsed();
BucketT *B = getBuckets();
const unsigned Mask = getNumBuckets() - 1;
llvm::densemap::detail::forEachUsed(OtherU, E, [&](unsigned I) {
@@ -551,10 +551,10 @@ class DenseMapBase : public DebugEpochBase {
BucketT *Buckets = getBuckets();
const BucketT *OtherBuckets = other.getBuckets();
const unsigned NumBuckets = getNumBuckets();
- used_t *U = getUsed();
- const used_t *OtherU = other.getUsed();
+ UsedT *U = getUsed();
+ const UsedT *OtherU = other.getUsed();
std::memcpy(U, OtherU,
- llvm::densemap::detail::usedWords(NumBuckets) * sizeof(used_t));
+ llvm::densemap::detail::usedWords(NumBuckets) * sizeof(UsedT));
if constexpr (std::is_trivially_copyable_v<KeyT> &&
std::is_trivially_copyable_v<ValueT>) {
memcpy(reinterpret_cast<void *>(Buckets), OtherBuckets,
@@ -585,7 +585,7 @@ class DenseMapBase : public DebugEpochBase {
decrementNumEntries();
BucketT *BucketsPtr = getBuckets();
- used_t *U = getUsed();
+ UsedT *U = getUsed();
const unsigned Mask = getNumBuckets() - 1;
unsigned I = TheBucket - BucketsPtr;
unsigned J = I;
@@ -671,9 +671,9 @@ class DenseMapBase : public DebugEpochBase {
Rep getRep() const { return derived().getRep(); }
- const used_t *getUsed() const { return derived().getUsed(); }
+ const UsedT *getUsed() const { return derived().getUsed(); }
- used_t *getUsed() { return derived().getUsed(); }
+ UsedT *getUsed() { return derived().getUsed(); }
unsigned getNumBuckets() const { return derived().getNumBuckets(); }
@@ -835,10 +835,10 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
// Lift some types from the dependent base class into this class for
// simplicity of referring to them.
using BaseT = DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
- using used_t = llvm::densemap::detail::used_t;
+ using UsedT = llvm::densemap::detail::UsedT;
BucketT *Buckets = nullptr;
- used_t *Used = nullptr;
+ UsedT *Used = nullptr;
unsigned NumEntries = 0;
unsigned NumBuckets = 0;
@@ -904,7 +904,7 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
typename BaseT::Rep getRep() const { return {Buckets, Used, NumBuckets}; }
- used_t *getUsed() const { return Used; }
+ UsedT *getUsed() const { return Used; }
unsigned getNumBuckets() const { return NumBuckets; }
@@ -931,7 +931,7 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
allocate_buffer(llvm::densemap::detail::allocBytes<BucketT>(NumBuckets),
llvm::densemap::detail::allocAlign<BucketT>()));
Buckets = reinterpret_cast<BucketT *>(Storage);
- Used = reinterpret_cast<used_t *>(Storage + sizeof(BucketT) * NumBuckets);
+ Used = reinterpret_cast<UsedT *>(Storage + sizeof(BucketT) * NumBuckets);
return true;
}
@@ -974,7 +974,7 @@ class SmallDenseMap
// Lift some types from the dependent base class into this class for
// simplicity of referring to them.
using BaseT = DenseMapBase<SmallDenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
- using used_t = llvm::densemap::detail::used_t;
+ using UsedT = llvm::densemap::detail::UsedT;
static_assert(isPowerOf2_64(InlineBuckets),
"InlineBuckets must be a power of 2.");
@@ -989,11 +989,11 @@ class SmallDenseMap
// Inline storage: the bucket array followed by the parallel used words.
struct InlineRep {
alignas(BucketT) char Buckets[sizeof(BucketT) * InlineBuckets];
- used_t Used[InlineUsedWords];
+ UsedT Used[InlineUsedWords];
};
struct LargeRep {
BucketT *Buckets;
- used_t *Used;
+ UsedT *Used;
unsigned NumBuckets;
};
@@ -1069,7 +1069,7 @@ class SmallDenseMap
// Both inline: swap the live bucket contents slot by slot, then the used
// used words. Buckets are raw storage, so a value may only move in one
// direction when exactly one side is occupied.
- used_t *LU = getInlineUsed(), *RU = RHS.getInlineUsed();
+ UsedT *LU = getInlineUsed(), *RU = RHS.getInlineUsed();
BucketT *LB = getInlineBuckets(), *RB = RHS.getInlineBuckets();
for (unsigned I = 0; I != InlineBuckets; ++I) {
bool L = llvm::densemap::detail::used(LU, I);
@@ -1105,7 +1105,7 @@ class SmallDenseMap
LargeRep TmpRep = LargeSide.storage.Large;
LargeSide.Small = true;
{
- used_t *SU = SmallSide.getInlineUsed(), *LU = LargeSide.getInlineUsed();
+ UsedT *SU = SmallSide.getInlineUsed(), *LU = LargeSide.getInlineUsed();
BucketT *SB = SmallSide.getInlineBuckets(),
*LB = LargeSide.getInlineBuckets();
for (unsigned I = 0; I != InlineBuckets; ++I)
@@ -1139,12 +1139,12 @@ class SmallDenseMap
return reinterpret_cast<BucketT *>(storage.Inline.Buckets);
}
- const used_t *getInlineUsed() const {
+ const UsedT *getInlineUsed() const {
assert(Small);
return storage.Inline.Used;
}
- used_t *getInlineUsed() {
+ UsedT *getInlineUsed() {
assert(Small);
return storage.Inline.Used;
}
@@ -1165,12 +1165,12 @@ class SmallDenseMap
const_cast<const SmallDenseMap *>(this)->getBuckets());
}
- const used_t *getUsed() const {
+ const UsedT *getUsed() const {
return Small ? getInlineUsed() : storage.Large.Used;
}
- used_t *getUsed() {
- return const_cast<used_t *>(
+ UsedT *getUsed() {
+ return const_cast<UsedT *>(
const_cast<const SmallDenseMap *>(this)->getUsed());
}
@@ -1201,7 +1201,7 @@ class SmallDenseMap
allocate_buffer(llvm::densemap::detail::allocBytes<BucketT>(Num),
llvm::densemap::detail::allocAlign<BucketT>()));
storage.Large.Buckets = reinterpret_cast<BucketT *>(S);
- storage.Large.Used = reinterpret_cast<used_t *>(S + sizeof(BucketT) * Num);
+ storage.Large.Used = reinterpret_cast<UsedT *>(S + sizeof(BucketT) * Num);
storage.Large.NumBuckets = Num;
return true;
}
@@ -1255,7 +1255,7 @@ class DenseMapIterator : DebugEpochBase::HandleBase {
friend class DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, true>;
friend class DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, false>;
- using used_t = llvm::densemap::detail::used_t;
+ using UsedT = llvm::densemap::detail::UsedT;
public:
using difference_type = ptrdiff_t;
@@ -1274,10 +1274,10 @@ class DenseMapIterator : DebugEpochBase::HandleBase {
// The non-reversed bucket base and the parallel used array. They map a
// bucket back to its index so AdvancePastEmptyBuckets can consult the bits.
pointer Buckets = {};
- const used_t *Used = {};
+ const UsedT *Used = {};
DenseMapIterator(BucketItTy Pos, BucketItTy E, pointer BucketsBase,
- const used_t *U, const DebugEpochBase &Epoch)
+ const UsedT *U, const DebugEpochBase &Epoch)
: DebugEpochBase::HandleBase(&Epoch), Ptr(Pos), End(E),
Buckets(BucketsBase), Used(U) {
assert(isHandleInSync() && "invalid construction!");
@@ -1286,7 +1286,7 @@ class DenseMapIterator : DebugEpochBase::HandleBase {
public:
DenseMapIterator() = default;
- static DenseMapIterator makeBegin(pointer Buckets, const used_t *Used,
+ static DenseMapIterator makeBegin(pointer Buckets, const UsedT *Used,
unsigned NumBuckets, bool IsEmpty,
const DebugEpochBase &Epoch) {
// When the map is empty, avoid the overhead of advancing/retreating past
@@ -1299,7 +1299,7 @@ class DenseMapIterator : DebugEpochBase::HandleBase {
return Iter;
}
- static DenseMapIterator makeEnd(pointer Buckets, const used_t *Used,
+ static DenseMapIterator makeEnd(pointer Buckets, const UsedT *Used,
unsigned NumBuckets,
const DebugEpochBase &Epoch) {
auto R = maybeReverse(llvm::make_range(Buckets, Buckets + NumBuckets));
@@ -1307,7 +1307,7 @@ class DenseMapIterator : DebugEpochBase::HandleBase {
}
static DenseMapIterator makeIterator(pointer P, pointer Buckets,
- const used_t *Used, unsigned NumBuckets,
+ const UsedT *Used, unsigned NumBuckets,
const DebugEpochBase &Epoch) {
auto R = maybeReverse(llvm::make_range(Buckets, Buckets + NumBuckets));
constexpr int Offset = shouldReverseIterate<KeyT>() ? 1 : 0;
@@ -1378,7 +1378,7 @@ class DenseMapIterator : DebugEpochBase::HandleBase {
}
const size_t NW = llvm::densemap::detail::usedWords(N);
size_t W = I >> 5;
- used_t Bits = Used[W] & (~used_t(0) << (I & 31));
+ UsedT Bits = Used[W] & (~UsedT(0) << (I & 31));
while (Bits == 0) {
if (++W == NW) {
Ptr = End;
More information about the llvm-commits
mailing list