[llvm] blah (PR #226866)
Kazu Hirata via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 27 18:36:49 PDT 2026
https://github.com/kazutakahirata created https://github.com/llvm/llvm-project/pull/226866
None
>From 9150e73d48a2011cfaa4f0087e1b17f214fc3ba9 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Sun, 27 Sep 2026 15:44:49 -0700
Subject: [PATCH] blah
---
llvm/include/llvm/ADT/DenseMap.h | 1893 +++++++++++++++---------------
1 file changed, 921 insertions(+), 972 deletions(-)
diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h
index f1763dcfed62a..9c854f5590cca 100644
--- a/llvm/include/llvm/ADT/DenseMap.h
+++ b/llvm/include/llvm/ADT/DenseMap.h
@@ -192,1198 +192,1147 @@ LLVM_ABI void *growRelocatable(void *OldBuckets, const UsedT *OldUsed,
size_t BucketSize, size_t Align,
BucketHasher Hasher, bool FreeOld);
-} // namespace densemap::detail
-
-// Befriended below so DenseMapBase can expose its bucket-relocation callback
-// erase to ValueHandleBase, the only caller that caches bucket pointers.
-class ValueHandleBase;
-
-template <typename KeyT, typename ValueT,
- typename KeyInfoT = DenseMapInfo<KeyT>,
- typename Bucket = llvm::detail::DenseMapPair<KeyT, ValueT>,
- bool IsConst = false>
-class DenseMapIterator;
-
-template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
- typename BucketT>
-class DenseMapBase : public DebugEpochBase {
- template <typename T>
- using const_arg_type_t = typename const_pointer_or_const_ref<T>::type;
+// A snapshot of the three fields the hot lookup paths need. Fetching them
+// together lets SmallDenseMap test its Small discriminator once rather than
+// once per accessor; for plain DenseMap it is three member loads either way.
+template <typename BucketT> struct StorageRep {
+ const BucketT *Buckets;
+ const UsedT *Used;
+ unsigned NumBuckets;
+};
- using UsedT = llvm::densemap::detail::UsedT;
+template <typename BucketT> class DenseMapStorage {
+ BucketT *Buckets = nullptr;
+ UsedT *Used = nullptr;
+ unsigned NumEntries = 0;
+ unsigned NumBuckets = 0;
public:
- using size_type = unsigned;
- using key_type = KeyT;
- using mapped_type = ValueT;
- using value_type = BucketT;
+ DenseMapStorage() = default;
- using iterator = DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT>;
- using const_iterator =
- DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT, true>;
+ unsigned getNumEntries() const { return NumEntries; }
+ void setNumEntries(unsigned Num) { NumEntries = Num; }
- [[nodiscard]] inline iterator begin() {
- return iterator::makeBegin(getBuckets(), getUsed(), getNumBuckets(),
- empty(), *this);
+ BucketT *getBuckets() const { return Buckets; }
+ UsedT *getUsed() const { return Used; }
+ unsigned getNumBuckets() const { return NumBuckets; }
+ StorageRep<BucketT> getRep() const { return {Buckets, Used, NumBuckets}; }
+
+ void swap(DenseMapStorage &RHS) {
+ std::swap(Buckets, RHS.Buckets);
+ std::swap(Used, RHS.Used);
+ std::swap(NumEntries, RHS.NumEntries);
+ std::swap(NumBuckets, RHS.NumBuckets);
}
- [[nodiscard]] inline iterator end() {
- return iterator::makeEnd(getBuckets(), getUsed(), getNumBuckets(), *this);
+
+ void setStorage(void *Storage, unsigned Num) {
+ Buckets = static_cast<BucketT *>(Storage);
+ Used = usedFor(Storage, sizeof(BucketT), Num);
+ NumBuckets = Num;
}
- [[nodiscard]] inline const_iterator begin() const {
- return const_iterator::makeBegin(getBuckets(), getUsed(), getNumBuckets(),
- empty(), *this);
+
+ void growShared(unsigned MinNumBuckets, BucketHasher Hasher) {
+ unsigned NewNumBuckets = roundUpNumBuckets(MinNumBuckets);
+ setStorage(growRelocatable(Buckets, Used, NumBuckets, NewNumBuckets,
+ sizeof(BucketT), allocAlign<BucketT>(), Hasher,
+ /*FreeOld=*/true),
+ NewNumBuckets);
}
- [[nodiscard]] inline const_iterator end() const {
- return const_iterator::makeEnd(getBuckets(), getUsed(), getNumBuckets(),
- *this);
+
+ void deallocateBuckets() {
+ if (NumBuckets == 0)
+ return;
+ deallocate_buffer(Buckets, allocBytes<BucketT>(NumBuckets),
+ allocAlign<BucketT>());
+ Buckets = nullptr;
+ Used = nullptr;
+ NumBuckets = 0;
}
- // Return an iterator to iterate over keys in the map.
- [[nodiscard]] inline auto keys() {
- return map_range(*this, [](const BucketT &P) { return P.getFirst(); });
+ bool allocateBuckets(unsigned Num) {
+ if (Num == 0) {
+ Buckets = nullptr;
+ Used = nullptr;
+ NumBuckets = 0;
+ return false;
+ }
+ setStorage(allocate_buffer(allocBytes<BucketT>(Num), allocAlign<BucketT>()),
+ Num);
+ return true;
}
- // Return an iterator to iterate over values in the map.
- [[nodiscard]] inline auto values() {
- return map_range(*this, [](const BucketT &P) { return P.getSecond(); });
+ // Put the zombie instance in a known good state after a move.
+ // deallocateBuckets() already resets to the empty state.
+ void kill() { deallocateBuckets(); }
+
+ static unsigned roundUpNumBuckets(unsigned MinNumBuckets) {
+ return std::max(64u, MinNumBuckets);
}
- [[nodiscard]] inline auto keys() const {
- return map_range(*this, [](const BucketT &P) { return P.getFirst(); });
+ // Plan how to shrink the bucket table. Return:
+ // - {false, 0} to reuse the existing bucket table
+ // - {true, N} to reallocate a bucket table with N entries
+ std::pair<bool, unsigned> planShrinkAndClear() const {
+ unsigned NewNumBuckets = 0;
+ if (NumEntries)
+ NewNumBuckets = std::max(64u, 1u << (Log2_32_Ceil(NumEntries) + 1));
+ if (NewNumBuckets == NumBuckets)
+ return {false, 0}; // Reuse.
+ return {true, NewNumBuckets}; // Reallocate.
}
- [[nodiscard]] inline auto values() const {
- return map_range(*this, [](const BucketT &P) { return P.getSecond(); });
+ bool maybeMoveFast(DenseMapStorage &&Other) {
+ swap(Other);
+ return true;
}
+};
- [[nodiscard]] bool empty() const { return getNumEntries() == 0; }
- [[nodiscard]] unsigned size() const { return getNumEntries(); }
+template <typename BucketT, unsigned InlineBuckets = 4>
+class SmallDenseMapStorage {
+ static_assert(isPowerOf2_64(InlineBuckets),
+ "InlineBuckets must be a power of 2.");
- /// Grow the densemap so that it can contain at least \p NumEntries items
- /// before resizing again.
- void reserve(size_type NumEntries) {
- auto NumBuckets = getMinBucketToReserveForEntries(NumEntries);
- incrementEpoch();
- if (NumBuckets > getNumBuckets())
- grow(NumBuckets);
- }
+ // Number of used words backing the inline buckets (>= 1).
+ static constexpr unsigned InlineUsedWords = usedWords(InlineBuckets);
- void clear() {
- incrementEpoch();
- if (getNumEntries() == 0)
- return;
+ unsigned Small : 1;
+ unsigned NumEntries : 31;
- // If the capacity of the array is huge, and the # elements used is small,
- // shrink the array.
- if (getNumEntries() * 4 < getNumBuckets() && getNumBuckets() > 64) {
- shrink_and_clear();
- return;
- }
+ // Inline storage: the bucket array followed by the parallel used words.
+ struct InlineRep {
+ alignas(BucketT) char Buckets[sizeof(BucketT) * InlineBuckets];
+ UsedT Used[InlineUsedWords];
+ };
+ struct LargeRep {
+ BucketT *Buckets;
+ UsedT *Used;
+ unsigned NumBuckets;
+ };
- destroyAll();
- llvm::densemap::detail::clearUsed(getUsed(), getNumBuckets());
- setNumEntries(0);
- }
+ // Discriminated by the Small bit.
+ union {
+ InlineRep Inline;
+ LargeRep Large;
+ } storage;
- void shrink_and_clear() {
- auto [Reallocate, NewNumBuckets] = derived().planShrinkAndClear();
- destroyAll();
- if (!Reallocate) {
- initEmpty();
- return;
- }
- derived().deallocateBuckets();
- initWithExactBucketCount(NewNumBuckets);
+ // Move-construct *Dst from *Src, then destroy *Src. Dst is raw storage.
+ static void relocateBucket(BucketT *Dst, BucketT *Src) {
+ using KeyT = std::remove_reference_t<decltype(Dst->getFirst())>;
+ using ValueT = std::remove_reference_t<decltype(Dst->getSecond())>;
+ ::new (&Dst->getFirst()) KeyT(std::move(Src->getFirst()));
+ ::new (&Dst->getSecond()) ValueT(std::move(Src->getSecond()));
+ Src->getSecond().~ValueT();
+ Src->getFirst().~KeyT();
}
- /// Return true if the specified key is in the map, false otherwise.
- [[nodiscard]] bool contains(const_arg_type_t<KeyT> Val) const {
- return doFind(Val) != nullptr;
+ const BucketT *getInlineBuckets() const {
+ assert(Small);
+ // Note that this cast does not violate aliasing rules as we assert that
+ // the memory's dynamic type is the small, inline bucket buffer, and the
+ // 'storage' is a POD containing a char buffer.
+ return reinterpret_cast<const BucketT *>(storage.Inline.Buckets);
}
- /// Return 1 if the specified key is in the map, 0 otherwise.
- [[nodiscard]] size_type count(const_arg_type_t<KeyT> Val) const {
- return contains(Val) ? 1 : 0;
+ BucketT *getInlineBuckets() {
+ assert(Small);
+ return reinterpret_cast<BucketT *>(storage.Inline.Buckets);
}
- [[nodiscard]] iterator find(const_arg_type_t<KeyT> Val) {
- return find_as(Val);
- }
- [[nodiscard]] const_iterator find(const_arg_type_t<KeyT> Val) const {
- return find_as(Val);
+ const UsedT *getInlineUsed() const {
+ assert(Small);
+ return storage.Inline.Used;
}
- /// Alternate version of find() which allows a different, and possibly
- /// less expensive, key type.
- /// The DenseMapInfo is responsible for supplying methods
- /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key
- /// type used.
- template <class LookupKeyT>
- [[nodiscard]] iterator find_as(const LookupKeyT &Val) {
- if (BucketT *Bucket = doFind(Val))
- return makeIterator(Bucket);
- return end();
- }
- template <class LookupKeyT>
- [[nodiscard]] const_iterator find_as(const LookupKeyT &Val) const {
- if (const BucketT *Bucket = doFind(Val))
- return makeConstIterator(Bucket);
- return end();
+ UsedT *getInlineUsed() {
+ assert(Small);
+ return storage.Inline.Used;
}
- /// Return the entry for the specified key, or a default constructed value if
- /// no such entry exists.
- [[nodiscard]] ValueT lookup(const_arg_type_t<KeyT> Val) const {
- if (const BucketT *Bucket = doFind(Val))
- return Bucket->getSecond();
- return ValueT();
+ void setLarge(void *Storage, unsigned NumBuckets) {
+ Small = false;
+ storage.Large = {static_cast<BucketT *>(Storage),
+ usedFor(Storage, sizeof(BucketT), NumBuckets), NumBuckets};
}
- // Return the entry with the specified key, or \p Default. This variant is
- // useful, because `lookup` cannot be used with non-default-constructible
- // values.
- template <typename U = std::remove_cv_t<ValueT>>
- [[nodiscard]] ValueT lookup_or(const_arg_type_t<KeyT> Val,
- U &&Default) const {
- if (const BucketT *Bucket = doFind(Val))
- return Bucket->getSecond();
- return Default;
- }
+public:
+ SmallDenseMapStorage() = default;
- /// Return the entry for the specified key, or abort if no such entry exists.
- [[nodiscard]] ValueT &at(const_arg_type_t<KeyT> Val) {
- auto Iter = this->find(std::move(Val));
- assert(Iter != this->end() && "DenseMap::at failed due to a missing key");
- return Iter->second;
+ unsigned getNumEntries() const { return NumEntries; }
+
+ void setNumEntries(unsigned Num) {
+ // NumEntries is hardcoded to be 31 bits wide.
+ assert(Num < (1U << 31) && "Cannot support more than 1<<31 entries");
+ NumEntries = Num;
}
- /// Return the entry for the specified key, or abort if no such entry exists.
- [[nodiscard]] const ValueT &at(const_arg_type_t<KeyT> Val) const {
- auto Iter = this->find(std::move(Val));
- assert(Iter != this->end() && "DenseMap::at failed due to a missing key");
- return Iter->second;
+ const BucketT *getBuckets() const {
+ return Small ? getInlineBuckets() : storage.Large.Buckets;
}
- // Inserts key,value pair into the map if the key isn't already in the map.
- // If the key is already in the map, it returns false and doesn't update the
- // value.
- std::pair<iterator, bool> insert(const std::pair<KeyT, ValueT> &KV) {
- return try_emplace_impl(KV.first, KV.second);
+ BucketT *getBuckets() {
+ return const_cast<BucketT *>(
+ const_cast<const SmallDenseMapStorage *>(this)->getBuckets());
}
- // Inserts key,value pair into the map if the key isn't already in the map.
- // If the key is already in the map, it returns false and doesn't update the
- // value.
- std::pair<iterator, bool> insert(std::pair<KeyT, ValueT> &&KV) {
- return try_emplace_impl(std::move(KV.first), std::move(KV.second));
+ const UsedT *getUsed() const {
+ return Small ? getInlineUsed() : storage.Large.Used;
}
- template <
- typename B = BucketT,
- typename = std::enable_if_t<!std::is_same_v<B, std::pair<KeyT, ValueT>>>>
- std::pair<iterator, bool> insert(const BucketT &KV) {
- return try_emplace_impl(KV.first, KV.second);
+ UsedT *getUsed() {
+ return const_cast<UsedT *>(
+ const_cast<const SmallDenseMapStorage *>(this)->getUsed());
}
- template <
- typename B = BucketT,
- typename = std::enable_if_t<!std::is_same_v<B, std::pair<KeyT, ValueT>>>>
- std::pair<iterator, bool> insert(BucketT &&KV) {
- return try_emplace_impl(std::move(KV.first), std::move(KV.second));
- }
-
- // Inserts key,value pair into the map if the key isn't already in the map.
- // The value is constructed in-place if the key is not in the map, otherwise
- // it is not moved.
- template <typename... Ts>
- std::pair<iterator, bool> try_emplace(KeyT &&Key, Ts &&...Args) {
- return try_emplace_impl(std::move(Key), std::forward<Ts>(Args)...);
- }
-
- // Inserts key,value pair into the map if the key isn't already in the map.
- // The value is constructed in-place if the key is not in the map, otherwise
- // it is not moved.
- template <typename... Ts>
- std::pair<iterator, bool> try_emplace(const KeyT &Key, Ts &&...Args) {
- return try_emplace_impl(Key, std::forward<Ts>(Args)...);
- }
-
- /// Alternate version of insert() which allows a different, and possibly
- /// less expensive, key type.
- /// The DenseMapInfo is responsible for supplying methods
- /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key
- /// type used.
- template <typename LookupKeyT>
- std::pair<iterator, bool> insert_as(std::pair<KeyT, ValueT> &&KV,
- const LookupKeyT &Val) {
- BucketT *TheBucket;
- if (LookupBucketFor(Val, TheBucket))
- return {makeIterator(TheBucket), false}; // Already in map.
-
- // Otherwise, insert the new element.
- TheBucket = findBucketForInsertion(Val, TheBucket);
- ::new (&TheBucket->getFirst()) KeyT(std::move(KV.first));
- ::new (&TheBucket->getSecond()) ValueT(std::move(KV.second));
- return {makeIterator(TheBucket), true};
- }
-
- /// Range insertion of pairs.
- template <typename InputIt> void insert(InputIt I, InputIt E) {
- for (; I != E; ++I)
- insert(*I);
- }
-
- /// Inserts range of 'std::pair<KeyT, ValueT>' values into the map.
- template <typename Range> void insert_range(Range &&R) {
- insert(adl_begin(R), adl_end(R));
- }
-
- template <typename V>
- std::pair<iterator, bool> insert_or_assign(const KeyT &Key, V &&Val) {
- auto Ret = try_emplace(Key, std::forward<V>(Val));
- if (!Ret.second)
- Ret.first->second = std::forward<V>(Val);
- return Ret;
- }
-
- template <typename V>
- std::pair<iterator, bool> insert_or_assign(KeyT &&Key, V &&Val) {
- auto Ret = try_emplace(std::move(Key), std::forward<V>(Val));
- if (!Ret.second)
- Ret.first->second = std::forward<V>(Val);
- return Ret;
- }
-
- template <typename... Ts>
- std::pair<iterator, bool> emplace_or_assign(const KeyT &Key, Ts &&...Args) {
- auto Ret = try_emplace(Key, std::forward<Ts>(Args)...);
- if (!Ret.second)
- Ret.first->second = ValueT(std::forward<Ts>(Args)...);
- return Ret;
- }
-
- template <typename... Ts>
- std::pair<iterator, bool> emplace_or_assign(KeyT &&Key, Ts &&...Args) {
- auto Ret = try_emplace(std::move(Key), std::forward<Ts>(Args)...);
- if (!Ret.second)
- Ret.first->second = ValueT(std::forward<Ts>(Args)...);
- return Ret;
+ unsigned getNumBuckets() const {
+ return Small ? InlineBuckets : storage.Large.NumBuckets;
}
- void eraseFromFilledBucket(BucketT *TheBucket) {
- eraseFromFilledBucket(TheBucket, [](BucketT &) {});
+ StorageRep<BucketT> getRep() const {
+ if (Small)
+ return {getInlineBuckets(), getInlineUsed(), InlineBuckets};
+ return {storage.Large.Buckets, storage.Large.Used,
+ storage.Large.NumBuckets};
}
- bool erase(const KeyT &Val) {
- BucketT *TheBucket = doFind(Val);
- if (!TheBucket)
- return false; // not in map.
-
- eraseFromFilledBucket(TheBucket);
- return true;
- }
- void erase(iterator I) { eraseFromFilledBucket(&*I); }
+ void swap(SmallDenseMapStorage &RHS) {
+ unsigned TmpNumEntries = RHS.NumEntries;
+ RHS.NumEntries = NumEntries;
+ NumEntries = TmpNumEntries;
- /// Remove entries that match the given predicate. \p Pred is invoked
- /// with a reference to each live bucket and must not access the map being
- /// modified. This is the safe replacement for erase-while-iterating.
- ///
- /// Returns whether anything was removed. If so, all iterators and references
- /// into the map are invalidated.
- template <typename Predicate> bool remove_if(Predicate Pred) {
- UsedT *U = getUsed();
- unsigned NumBuckets = getNumBuckets();
- BucketT *B = getBuckets();
- bool Removed = false;
- for (unsigned I = 0; I != NumBuckets; ++I) {
- if (!llvm::densemap::detail::used(U, I))
- continue;
- if (Pred(B[I])) {
- B[I].getSecond().~ValueT();
- B[I].getFirst().~KeyT();
- llvm::densemap::detail::unsetUsed(U, I);
- decrementNumEntries();
- Removed = true;
+ if (Small && RHS.Small) {
+ // Both inline: swap the live bucket contents slot by slot, then the used
+ // words. Buckets are raw storage, so a value may only move in one
+ // direction when exactly one side is occupied.
+ UsedT *LU = getInlineUsed(), *RU = RHS.getInlineUsed();
+ BucketT *LB = getInlineBuckets(), *RB = RHS.getInlineBuckets();
+ for (unsigned I = 0; I != InlineBuckets; ++I) {
+ bool L = used(LU, I);
+ bool R = used(RU, I);
+ if (L && R) {
+ // Both occupied: exchange through a temporary.
+ alignas(BucketT) char Tmp[sizeof(BucketT)];
+ BucketT *T = reinterpret_cast<BucketT *>(Tmp);
+ relocateBucket(T, &LB[I]);
+ relocateBucket(&LB[I], &RB[I]);
+ relocateBucket(&RB[I], T);
+ } else if (L) {
+ relocateBucket(&RB[I], &LB[I]);
+ } else if (R) {
+ relocateBucket(&LB[I], &RB[I]);
+ }
}
- }
- if (Removed) {
- incrementEpoch();
- this->grow(NumBuckets);
- }
- return Removed;
- }
-
- ValueT &operator[](const KeyT &Key) {
- return lookupOrInsertIntoBucket(Key).first->second;
- }
-
- ValueT &operator[](KeyT &&Key) {
- return lookupOrInsertIntoBucket(std::move(Key)).first->second;
- }
-
- /// Return true if the specified pointer points somewhere into the DenseMap's
- /// array of buckets (i.e. either to a key or value in the DenseMap).
- [[nodiscard]] bool isPointerIntoBucketsArray(const void *Ptr) const {
- return Ptr >= getBuckets() && Ptr < getBucketsEnd();
- }
-
- /// getPointerIntoBucketsArray() - Return an opaque pointer into the buckets
- /// array. In conjunction with the previous method, this can be used to
- /// determine whether an insertion caused the DenseMap to reallocate.
- [[nodiscard]] const void *getPointerIntoBucketsArray() const {
- return getBuckets();
- }
-
- void swap(DerivedT &RHS) {
- this->incrementEpoch();
- RHS.incrementEpoch();
- derived().swapImpl(RHS);
- }
-
-protected:
- DenseMapBase() = default;
-
- struct ExactBucketCount {};
-
- // A snapshot of the three fields the hot lookup paths need. Fetching them
- // together lets SmallDenseMap test its Small discriminator once rather than
- // once per accessor; for plain DenseMap it is three member loads either way.
- struct Rep {
- const BucketT *Buckets;
- const UsedT *Used;
- unsigned NumBuckets;
- };
-
- void initWithExactBucketCount(unsigned NewNumBuckets) {
- if (derived().allocateBuckets(NewNumBuckets))
- initEmpty();
- else
- setNumEntries(0);
- }
-
- void destroyAll() {
- // No need to iterate through the buckets if the bucket is trivially
- // destructible.
- if constexpr (std::is_trivially_destructible_v<BucketT>)
- return;
-
- if (getNumBuckets() == 0) // Nothing to do.
+ for (unsigned W = 0; W != InlineUsedWords; ++W)
+ std::swap(LU[W], RU[W]);
return;
-
- BucketT *B = getBuckets();
- const UsedT *U = getUsed();
- const unsigned E = getNumBuckets();
- llvm::densemap::detail::forEachUsed(U, E, [&](unsigned I) {
- B[I].getSecond().~ValueT();
- B[I].getFirst().~KeyT();
- });
- }
-
- void initEmpty() {
- static_assert(std::is_base_of_v<DenseMapBase, DerivedT>,
- "Must pass the derived type to this template!");
- setNumEntries(0);
-
- assert((getNumBuckets() & (getNumBuckets() - 1)) == 0 &&
- "# initial buckets must be a power of two!");
- if (getNumBuckets())
- llvm::densemap::detail::clearUsed(getUsed(), getNumBuckets());
- }
-
- /// Returns the number of buckets to allocate to ensure that the DenseMap can
- /// accommodate \p NumEntries without need to grow().
- unsigned getMinBucketToReserveForEntries(unsigned NumEntries) {
- // Ensure that "NumEntries * 4 < NumBuckets * 3"
- if (NumEntries == 0)
- return 0;
- // +1 is required because of the strict inequality.
- // For example, if NumEntries is 48, we need to return 128.
- return NextPowerOf2(NumEntries * 4 / 3 + 1);
- }
-
- static constexpr llvm::densemap::detail::BucketHasher hasher() {
- return llvm::densemap::detail::hasherFor<KeyT, KeyInfoT>();
- }
-
- // Move key/value from Other to *this.
- // Other is left in a valid but empty state.
- LLVM_ATTRIBUTE_NOINLINE void moveFrom(DerivedT &Other) {
- assert(getNumEntries() == 0 && "moveFrom requires an empty destination");
- BucketT *OtherB = Other.getBuckets();
- UsedT *OtherU = Other.getUsed();
- const unsigned E = Other.getNumBuckets();
- UsedT *U = getUsed();
- BucketT *B = getBuckets();
- const unsigned Mask = getNumBuckets() - 1;
- llvm::densemap::detail::forEachUsed(OtherU, E, [&](unsigned I) {
- // Find the first empty slot on this key's probe chain; there is no equal
- // key in the destination, so nothing to compare against.
- unsigned BucketNo = KeyInfoT::getHashValue(OtherB[I].getFirst()) & Mask;
- while (llvm::densemap::detail::used(U, BucketNo))
- BucketNo = (BucketNo + 1) & Mask;
- BucketT *DestBucket = B + BucketNo;
- ::new (&DestBucket->getFirst()) KeyT(std::move(OtherB[I].getFirst()));
- ::new (&DestBucket->getSecond()) ValueT(std::move(OtherB[I].getSecond()));
- llvm::densemap::detail::setUsed(U, BucketNo);
-
- // Free the moved-out key/value.
- OtherB[I].getSecond().~ValueT();
- OtherB[I].getFirst().~KeyT();
- });
- setNumEntries(Other.getNumEntries());
- Other.derived().kill();
- }
-
- LLVM_ATTRIBUTE_NOINLINE void copyFrom(const DerivedT &other) {
- this->destroyAll();
- derived().deallocateBuckets();
- setNumEntries(0);
- if (!derived().allocateBuckets(other.getNumBuckets())) {
- // The bucket list is empty. No work to do.
+ }
+ if (!Small && !RHS.Small) {
+ std::swap(storage.Large, RHS.storage.Large);
return;
}
- assert(&other != this);
- assert(getNumBuckets() == other.getNumBuckets());
-
- setNumEntries(other.getNumEntries());
+ SmallDenseMapStorage &SmallSide = Small ? *this : RHS;
+ SmallDenseMapStorage &LargeSide = Small ? RHS : *this;
- BucketT *Buckets = getBuckets();
- const BucketT *OtherBuckets = other.getBuckets();
- const unsigned NumBuckets = getNumBuckets();
- UsedT *U = getUsed();
- const UsedT *OtherU = other.getUsed();
- std::memcpy(U, OtherU,
- llvm::densemap::detail::usedWords(NumBuckets) * sizeof(UsedT));
- if constexpr (densemap::detail::isRelocatableBucket<BucketT>) {
- memcpy(reinterpret_cast<void *>(Buckets), OtherBuckets,
- NumBuckets * sizeof(BucketT));
- } else {
- llvm::densemap::detail::forEachUsed(U, NumBuckets, [&](unsigned I) {
- ::new (&Buckets[I].getFirst()) KeyT(OtherBuckets[I].getFirst());
- ::new (&Buckets[I].getSecond()) ValueT(OtherBuckets[I].getSecond());
- });
+ // Stash the large rep, then move the small side's inline contents into the
+ // large side (which becomes inline), and finally install the rep on the
+ // small side (which becomes large).
+ LargeRep TmpRep = LargeSide.storage.Large;
+ LargeSide.Small = true;
+ {
+ UsedT *SU = SmallSide.getInlineUsed(), *LU = LargeSide.getInlineUsed();
+ BucketT *SB = SmallSide.getInlineBuckets(),
+ *LB = LargeSide.getInlineBuckets();
+ for (unsigned I = 0; I != InlineBuckets; ++I)
+ if (used(SU, I))
+ relocateBucket(&LB[I], &SB[I]);
+ for (unsigned W = 0; W != InlineUsedWords; ++W)
+ LU[W] = SU[W];
}
+ SmallSide.Small = false;
+ SmallSide.storage.Large = TmpRep;
}
-private:
- // ValueHandleBase caches pointers into the bucket array, so it needs the
- // callback erase below to fix them up as entries shift. It is the only
- // intended caller; do not add new ones.
- friend class ValueHandleBase;
-
- /// Erase the entry at \p TheBucket and close the resulting hole via Knuth
- /// TAOCP 6.4 Algorithm R. For callers that cache pointers into the bucket
- /// array, call \p OnMoved per shifted bucket.
- template <typename OnMovedT>
- LLVM_ATTRIBUTE_NOINLINE void eraseFromFilledBucket(BucketT *TheBucket,
- OnMovedT &&OnMoved) {
- incrementEpoch();
- TheBucket->getSecond().~ValueT();
- TheBucket->getFirst().~KeyT();
- decrementNumEntries();
-
- BucketT *BucketsPtr = getBuckets();
- UsedT *U = getUsed();
- const unsigned Mask = getNumBuckets() - 1;
- unsigned I = TheBucket - BucketsPtr;
- unsigned J = I;
- while (true) {
- J = (J + 1) & Mask;
- BucketT &BJ = BucketsPtr[J];
- if (!llvm::densemap::detail::used(U, J))
- break;
- auto Ideal = KeyInfoT::getHashValue(BJ.getFirst());
- // If the hole (I) lies on the linear-probe chain from the home bucket
- // (Ideal) to J, shift J into the hole and make J the new hole.
- if (((I - Ideal) & Mask) < ((J - Ideal) & Mask)) {
- BucketT &BI = BucketsPtr[I];
- ::new (&BI.getFirst()) KeyT(std::move(BJ.getFirst()));
- ::new (&BI.getSecond()) ValueT(std::move(BJ.getSecond()));
- BJ.getSecond().~ValueT();
- BJ.getFirst().~KeyT();
- OnMoved(BI);
- I = J;
- }
+ void growShared(unsigned MinNumBuckets, BucketHasher Hasher) {
+ unsigned NewNumBuckets = roundUpNumBuckets(MinNumBuckets);
+ // remove_if asks for the count it already has: rehash in place.
+ if (Small && NewNumBuckets <= InlineBuckets) {
+ InlineRep Old = storage.Inline;
+ clearUsed(getInlineUsed(), InlineBuckets);
+ rehashRelocatable(getInlineBuckets(), getInlineUsed(), InlineBuckets,
+ Old.Buckets, Old.Used, InlineBuckets, sizeof(BucketT),
+ Hasher);
+ return;
}
- llvm::densemap::detail::unsetUsed(U, I);
+ void *Storage = growRelocatable(
+ getBuckets(), getUsed(), getNumBuckets(), NewNumBuckets,
+ sizeof(BucketT), allocAlign<BucketT>(), Hasher, /*FreeOld=*/!Small);
+ setLarge(Storage, NewNumBuckets);
}
- /// Erase \p Val and close the resulting hole by potentially shifting other
- /// entries into it. For callers that cache pointers into the bucket array,
- /// call \p OnMoved per shifted bucket.
- template <typename OnMovedT> bool erase(const KeyT &Val, OnMovedT &&OnMoved) {
- BucketT *TheBucket = doFind(Val);
- if (!TheBucket)
- return false;
- eraseFromFilledBucket(TheBucket, std::forward<OnMovedT>(OnMoved));
- return true;
- }
+ void deallocateBuckets() {
+ // Fast path in case storage.Large.NumBuckets == 0, just like destroyAll.
+ // This path is used to destruct zombie instances after moves.
+ if (Small || storage.Large.NumBuckets == 0)
+ return;
- DerivedT &derived() { return *static_cast<DerivedT *>(this); }
- const DerivedT &derived() const {
- return *static_cast<const DerivedT *>(this);
+ deallocate_buffer(storage.Large.Buckets,
+ allocBytes<BucketT>(storage.Large.NumBuckets),
+ allocAlign<BucketT>());
+ storage.Large.NumBuckets = 0;
}
- template <typename KeyArgT, typename... Ts>
- std::pair<BucketT *, bool> lookupOrInsertIntoBucket(KeyArgT &&Key,
- Ts &&...Args) {
- BucketT *TheBucket = nullptr;
- if (LookupBucketFor(Key, TheBucket))
- return {TheBucket, false}; // Already in the map.
-
- // Otherwise, insert the new element.
- TheBucket = findBucketForInsertion(Key, TheBucket);
- ::new (&TheBucket->getFirst()) KeyT(std::forward<KeyArgT>(Key));
- ::new (&TheBucket->getSecond()) ValueT(std::forward<Ts>(Args)...);
- return {TheBucket, true};
+ bool allocateBuckets(unsigned Num) {
+ if (Num <= InlineBuckets) {
+ Small = true;
+ return true;
+ }
+ setLarge(allocate_buffer(allocBytes<BucketT>(Num), allocAlign<BucketT>()),
+ Num);
+ return true;
}
- template <typename KeyArgT, typename... Ts>
- std::pair<iterator, bool> try_emplace_impl(KeyArgT &&Key, Ts &&...Args) {
- auto [Bucket, Inserted] = lookupOrInsertIntoBucket(
- std::forward<KeyArgT>(Key), std::forward<Ts>(Args)...);
- return {makeIterator(Bucket), Inserted};
+ // Put the zombie instance in a known good state after a move.
+ void kill() {
+ deallocateBuckets();
+ Small = false;
+ storage.Large = LargeRep{nullptr, nullptr, 0};
}
- iterator makeIterator(BucketT *TheBucket) {
- return iterator::makeIterator(TheBucket, getBuckets(), getUsed(),
- getNumBuckets(), *this);
+ static unsigned roundUpNumBuckets(unsigned MinNumBuckets) {
+ if (MinNumBuckets <= InlineBuckets)
+ return InlineBuckets;
+ return std::max(64u, MinNumBuckets);
}
- const_iterator makeConstIterator(const BucketT *TheBucket) const {
- return const_iterator::makeIterator(TheBucket, getBuckets(), getUsed(),
- getNumBuckets(), *this);
+ // Plan how to shrink the bucket table. Return:
+ // - {false, 0} to reuse the existing bucket table
+ // - {true, N} to reallocate a bucket table with N entries
+ std::pair<bool, unsigned> planShrinkAndClear() const {
+ unsigned NewNumBuckets = 0;
+ if (NumEntries) {
+ NewNumBuckets = 1u << (Log2_32_Ceil(NumEntries) + 1);
+ if (NewNumBuckets > InlineBuckets)
+ NewNumBuckets = std::max(64u, NewNumBuckets);
+ }
+ bool Reuse = Small ? NewNumBuckets <= InlineBuckets
+ : NewNumBuckets == storage.Large.NumBuckets;
+ if (Reuse)
+ return {false, 0}; // Reuse.
+ return {true, NewNumBuckets}; // Reallocate.
}
- unsigned getNumEntries() const { return derived().getNumEntries(); }
-
- void setNumEntries(unsigned Num) { derived().setNumEntries(Num); }
-
- void incrementNumEntries() { setNumEntries(getNumEntries() + 1); }
+ bool maybeMoveFast(SmallDenseMapStorage &&Other) {
+ if (Other.Small)
+ return false;
- void decrementNumEntries() { setNumEntries(getNumEntries() - 1); }
+ Small = false;
+ NumEntries = Other.NumEntries;
+ storage.Large = Other.storage.Large;
+ Other.storage.Large.NumBuckets = 0;
+ return true;
+ }
+};
- const BucketT *getBuckets() const { return derived().getBuckets(); }
+} // namespace densemap::detail
- BucketT *getBuckets() { return derived().getBuckets(); }
+// Befriended below so DenseMapBase can expose its bucket-relocation callback
+// erase to ValueHandleBase, the only caller that caches bucket pointers.
+class ValueHandleBase;
- Rep getRep() const { return derived().getRep(); }
+template <typename KeyT, typename ValueT,
+ typename KeyInfoT = DenseMapInfo<KeyT>,
+ typename Bucket = llvm::detail::DenseMapPair<KeyT, ValueT>,
+ bool IsConst = false>
+class DenseMapIterator;
- const UsedT *getUsed() const { return derived().getUsed(); }
+template <typename StorageT, typename KeyT, typename ValueT, typename KeyInfoT,
+ typename BucketT>
+class DenseMapBase : public DebugEpochBase {
+ template <typename T>
+ using const_arg_type_t = typename const_pointer_or_const_ref<T>::type;
- UsedT *getUsed() { return derived().getUsed(); }
+ using UsedT = llvm::densemap::detail::UsedT;
- unsigned getNumBuckets() const { return derived().getNumBuckets(); }
+public:
+ using size_type = unsigned;
+ using key_type = KeyT;
+ using mapped_type = ValueT;
+ using value_type = BucketT;
- BucketT *getBucketsEnd() { return getBuckets() + getNumBuckets(); }
+ using iterator = DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT>;
+ using const_iterator =
+ DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT, true>;
- const BucketT *getBucketsEnd() const {
- return getBuckets() + getNumBuckets();
+ [[nodiscard]] inline iterator begin() {
+ return iterator::makeBegin(getBuckets(), getUsed(), getNumBuckets(),
+ empty(), *this);
}
-
- LLVM_ATTRIBUTE_NOINLINE void grow(unsigned MinNumBuckets) {
- assert((MinNumBuckets == 0 || isPowerOf2_32(MinNumBuckets)) &&
- "bucket count must be zero or a power of two");
- if constexpr (llvm::densemap::detail::isRelocatableBucket<BucketT>) {
- derived().growShared(MinNumBuckets);
- } else {
- unsigned NumBuckets = DerivedT::roundUpNumBuckets(MinNumBuckets);
- DerivedT Tmp(NumBuckets, ExactBucketCount{});
- Tmp.moveFrom(derived());
- if (derived().maybeMoveFast(std::move(Tmp)))
- return;
- initWithExactBucketCount(NumBuckets);
- moveFrom(Tmp);
- }
+ [[nodiscard]] inline iterator end() {
+ return iterator::makeEnd(getBuckets(), getUsed(), getNumBuckets(), *this);
}
-
- template <typename LookupKeyT>
- BucketT *findBucketForInsertion(const LookupKeyT &Lookup,
- BucketT *TheBucket) {
- incrementEpoch();
-
- // Grow the table if the load factor would exceed 3/4 after insertion.
- // Linear probing with gap-closing deletion (Knuth Algorithm R) keeps
- // every chain compact and bounded by the table's empty-bucket count,
- // so no tombstone-driven resize is needed.
- unsigned NewNumEntries = getNumEntries() + 1;
- unsigned NumBuckets = getNumBuckets();
- if (LLVM_UNLIKELY(NewNumEntries * 4 >= NumBuckets * 3)) {
- this->grow(NumBuckets * 2);
- LookupBucketFor(Lookup, TheBucket);
- }
- assert(TheBucket);
-
- // Mark used. The caller will placement-construct the raw key/value.
- llvm::densemap::detail::setUsed(getUsed(), TheBucket - getBuckets());
-
- // Only update the state after we've grown our bucket space appropriately
- // so that when growing buckets we have self-consistent entry count.
- incrementNumEntries();
- return TheBucket;
+ [[nodiscard]] inline const_iterator begin() const {
+ return const_iterator::makeBegin(getBuckets(), getUsed(), getNumBuckets(),
+ empty(), *this);
}
-
- template <typename LookupKeyT>
- const BucketT *doFind(const LookupKeyT &Val) const {
- if (empty())
- return nullptr;
- auto [BucketsPtr, U, NumBuckets] = getRep();
-
- const unsigned Mask = NumBuckets - 1;
- unsigned BucketNo = KeyInfoT::getHashValue(Val) & Mask;
- while (true) {
- // An empty bucket terminates the probe: the key isn't in the map.
- if (LLVM_LIKELY(!llvm::densemap::detail::used(U, BucketNo)))
- return nullptr;
- const BucketT *Bucket = BucketsPtr + BucketNo;
- if (LLVM_LIKELY(KeyInfoT::isEqual(Val, Bucket->getFirst())))
- return Bucket;
-
- // Hash collision: continue linear probing.
- BucketNo = (BucketNo + 1) & Mask;
- }
+ [[nodiscard]] inline const_iterator end() const {
+ return const_iterator::makeEnd(getBuckets(), getUsed(), getNumBuckets(),
+ *this);
}
- template <typename LookupKeyT> BucketT *doFind(const LookupKeyT &Val) {
- return const_cast<BucketT *>(
- static_cast<const DenseMapBase *>(this)->doFind(Val));
+ // Return an iterator to iterate over keys in the map.
+ [[nodiscard]] inline auto keys() {
+ return map_range(*this, [](const BucketT &P) { return P.getFirst(); });
}
- /// Lookup the appropriate bucket for Val, returning it in FoundBucket. If the
- /// bucket contains the key and a value, this returns true, otherwise it
- /// returns a bucket with an empty marker and returns false.
- template <typename LookupKeyT>
- bool LookupBucketFor(const LookupKeyT &Val, BucketT *&FoundBucket) {
- auto [CBuckets, U, NumBuckets] = getRep();
- if (NumBuckets == 0) {
- FoundBucket = nullptr;
- return false;
- }
- // getRep() yields const pointers; this object is non-const, so recovering
- // a mutable bucket pointer is safe (mirrors the non-const getBuckets()).
- BucketT *BucketsPtr = const_cast<BucketT *>(CBuckets);
-
- const unsigned Mask = NumBuckets - 1;
- unsigned BucketNo = KeyInfoT::getHashValue(Val) & Mask;
- while (true) {
- BucketT *ThisBucket = BucketsPtr + BucketNo;
- // If we found an empty bucket, the key doesn't exist in the set.
- // Return it as the insertion point.
- if (LLVM_LIKELY(!llvm::densemap::detail::used(U, BucketNo))) {
- FoundBucket = ThisBucket;
- return false;
- }
-
- // Found Val's bucket? If so, return it.
- if (LLVM_LIKELY(KeyInfoT::isEqual(Val, ThisBucket->getFirst()))) {
- FoundBucket = ThisBucket;
- return true;
- }
+ // Return an iterator to iterate over values in the map.
+ [[nodiscard]] inline auto values() {
+ return map_range(*this, [](const BucketT &P) { return P.getSecond(); });
+ }
- // Hash collision: continue linear probing.
- BucketNo = (BucketNo + 1) & Mask;
- }
+ [[nodiscard]] inline auto keys() const {
+ return map_range(*this, [](const BucketT &P) { return P.getFirst(); });
}
-public:
- /// Return the approximate size (in bytes) of the actual map.
- /// This is just the raw memory used by DenseMap.
- /// If entries are pointers to objects, the size of the referenced objects
- /// are not included.
- [[nodiscard]] size_t getMemorySize() const {
- return llvm::densemap::detail::allocBytes<BucketT>(getNumBuckets());
+ [[nodiscard]] inline auto values() const {
+ return map_range(*this, [](const BucketT &P) { return P.getSecond(); });
}
-};
-/// Equality comparison for DenseMap.
-///
-/// Iterates over elements of LHS confirming that each (key, value) pair in LHS
-/// is also in RHS, and that no additional pairs are in RHS.
-/// Equivalent to N calls to RHS.find and N value comparisons. Amortized
-/// complexity is linear, worst case is O(N^2) (if every hash collides).
-template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
- typename BucketT>
-[[nodiscard]] bool
-operator==(const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS,
- const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) {
- if (LHS.size() != RHS.size())
- return false;
+ [[nodiscard]] bool empty() const { return getNumEntries() == 0; }
+ [[nodiscard]] unsigned size() const { return getNumEntries(); }
- for (auto &KV : LHS) {
- auto I = RHS.find(KV.first);
- if (I == RHS.end() || I->second != KV.second)
- return false;
+ /// Grow the densemap so that it can contain at least \p NumEntries items
+ /// before resizing again.
+ void reserve(size_type NumEntries) {
+ auto NumBuckets = getMinBucketToReserveForEntries(NumEntries);
+ incrementEpoch();
+ if (NumBuckets > getNumBuckets())
+ grow(NumBuckets);
}
- return true;
-}
-
-/// Inequality comparison for DenseMap.
-///
-/// Equivalent to !(LHS == RHS). See operator== for performance notes.
-template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
- typename BucketT>
-[[nodiscard]] bool
-operator!=(const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS,
- const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) {
- return !(LHS == RHS);
-}
+ void clear() {
+ incrementEpoch();
+ if (getNumEntries() == 0)
+ return;
-template <typename KeyT, typename ValueT,
- typename KeyInfoT = DenseMapInfo<KeyT>,
- typename BucketT = llvm::detail::DenseMapPair<KeyT, ValueT>>
-class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
- KeyT, ValueT, KeyInfoT, BucketT> {
- friend class DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
+ // If the capacity of the array is huge, and the # elements used is small,
+ // shrink the array.
+ if (getNumEntries() * 4 < getNumBuckets() && getNumBuckets() > 64) {
+ shrink_and_clear();
+ return;
+ }
- // Lift some types from the dependent base class into this class for
- // simplicity of referring to them.
- using BaseT = DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
- using UsedT = llvm::densemap::detail::UsedT;
+ destroyAll();
+ llvm::densemap::detail::clearUsed(getUsed(), getNumBuckets());
+ setNumEntries(0);
+ }
- BucketT *Buckets = nullptr;
- UsedT *Used = nullptr;
- unsigned NumEntries = 0;
- unsigned NumBuckets = 0;
+ void shrink_and_clear() {
+ auto [Reallocate, NewNumBuckets] = Storage.planShrinkAndClear();
+ destroyAll();
+ if (!Reallocate) {
+ initEmpty();
+ return;
+ }
+ Storage.deallocateBuckets();
+ initWithExactBucketCount(NewNumBuckets);
+ }
- explicit DenseMap(unsigned NumBuckets, typename BaseT::ExactBucketCount) {
- this->initWithExactBucketCount(NumBuckets);
+ /// Return true if the specified key is in the map, false otherwise.
+ [[nodiscard]] bool contains(const_arg_type_t<KeyT> Val) const {
+ return doFind(Val) != nullptr;
}
-public:
- /// Create a DenseMap with an optional \p NumElementsToReserve to guarantee
- /// that this number of elements can be inserted in the map without grow().
- explicit DenseMap(unsigned NumElementsToReserve = 0)
- : DenseMap(BaseT::getMinBucketToReserveForEntries(NumElementsToReserve),
- typename BaseT::ExactBucketCount{}) {}
+ /// Return 1 if the specified key is in the map, 0 otherwise.
+ [[nodiscard]] size_type count(const_arg_type_t<KeyT> Val) const {
+ return contains(Val) ? 1 : 0;
+ }
- DenseMap(const DenseMap &other) : DenseMap() { this->copyFrom(other); }
+ [[nodiscard]] iterator find(const_arg_type_t<KeyT> Val) {
+ return find_as(Val);
+ }
+ [[nodiscard]] const_iterator find(const_arg_type_t<KeyT> Val) const {
+ return find_as(Val);
+ }
- DenseMap(DenseMap &&other) : DenseMap() { this->swap(other); }
+ /// Alternate version of find() which allows a different, and possibly
+ /// less expensive, key type.
+ /// The DenseMapInfo is responsible for supplying methods
+ /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key
+ /// type used.
+ template <class LookupKeyT>
+ [[nodiscard]] iterator find_as(const LookupKeyT &Val) {
+ if (BucketT *Bucket = doFind(Val))
+ return makeIterator(Bucket);
+ return end();
+ }
+ template <class LookupKeyT>
+ [[nodiscard]] const_iterator find_as(const LookupKeyT &Val) const {
+ if (const BucketT *Bucket = doFind(Val))
+ return makeConstIterator(Bucket);
+ return end();
+ }
- template <typename InputIt>
- DenseMap(const InputIt &I, const InputIt &E) : DenseMap(std::distance(I, E)) {
- this->insert(I, E);
+ /// Return the entry for the specified key, or a default constructed value if
+ /// no such entry exists.
+ [[nodiscard]] ValueT lookup(const_arg_type_t<KeyT> Val) const {
+ if (const BucketT *Bucket = doFind(Val))
+ return Bucket->getSecond();
+ return ValueT();
}
- template <typename RangeT>
- DenseMap(llvm::from_range_t, const RangeT &Range)
- : DenseMap(adl_begin(Range), adl_end(Range)) {}
+ // Return the entry with the specified key, or \p Default. This variant is
+ // useful, because `lookup` cannot be used with non-default-constructible
+ // values.
+ template <typename U = std::remove_cv_t<ValueT>>
+ [[nodiscard]] ValueT lookup_or(const_arg_type_t<KeyT> Val,
+ U &&Default) const {
+ if (const BucketT *Bucket = doFind(Val))
+ return Bucket->getSecond();
+ return Default;
+ }
- DenseMap(std::initializer_list<typename BaseT::value_type> Vals)
- : DenseMap(Vals.begin(), Vals.end()) {}
+ /// Return the entry for the specified key, or abort if no such entry exists.
+ [[nodiscard]] ValueT &at(const_arg_type_t<KeyT> Val) {
+ auto Iter = this->find(std::move(Val));
+ assert(Iter != this->end() && "DenseMap::at failed due to a missing key");
+ return Iter->second;
+ }
- ~DenseMap() {
- this->destroyAll();
- deallocateBuckets();
+ /// Return the entry for the specified key, or abort if no such entry exists.
+ [[nodiscard]] const ValueT &at(const_arg_type_t<KeyT> Val) const {
+ auto Iter = this->find(std::move(Val));
+ assert(Iter != this->end() && "DenseMap::at failed due to a missing key");
+ return Iter->second;
}
- DenseMap &operator=(const DenseMap &other) {
- if (&other != this)
- this->copyFrom(other);
- return *this;
+ // Inserts key,value pair into the map if the key isn't already in the map.
+ // If the key is already in the map, it returns false and doesn't update the
+ // value.
+ std::pair<iterator, bool> insert(const std::pair<KeyT, ValueT> &KV) {
+ return try_emplace_impl(KV.first, KV.second);
}
- DenseMap &operator=(DenseMap &&other) {
- this->destroyAll();
- deallocateBuckets();
- this->initWithExactBucketCount(0);
- this->swap(other);
- return *this;
+ // Inserts key,value pair into the map if the key isn't already in the map.
+ // If the key is already in the map, it returns false and doesn't update the
+ // value.
+ std::pair<iterator, bool> insert(std::pair<KeyT, ValueT> &&KV) {
+ return try_emplace_impl(std::move(KV.first), std::move(KV.second));
}
-private:
- void swapImpl(DenseMap &RHS) {
- std::swap(Buckets, RHS.Buckets);
- std::swap(Used, RHS.Used);
- std::swap(NumEntries, RHS.NumEntries);
- std::swap(NumBuckets, RHS.NumBuckets);
+ template <
+ typename B = BucketT,
+ typename = std::enable_if_t<!std::is_same_v<B, std::pair<KeyT, ValueT>>>>
+ std::pair<iterator, bool> insert(const BucketT &KV) {
+ return try_emplace_impl(KV.first, KV.second);
}
- unsigned getNumEntries() const { return NumEntries; }
+ template <
+ typename B = BucketT,
+ typename = std::enable_if_t<!std::is_same_v<B, std::pair<KeyT, ValueT>>>>
+ std::pair<iterator, bool> insert(BucketT &&KV) {
+ return try_emplace_impl(std::move(KV.first), std::move(KV.second));
+ }
- void setNumEntries(unsigned Num) { NumEntries = Num; }
+ // Inserts key,value pair into the map if the key isn't already in the map.
+ // The value is constructed in-place if the key is not in the map, otherwise
+ // it is not moved.
+ template <typename... Ts>
+ std::pair<iterator, bool> try_emplace(KeyT &&Key, Ts &&...Args) {
+ return try_emplace_impl(std::move(Key), std::forward<Ts>(Args)...);
+ }
- BucketT *getBuckets() const { return Buckets; }
+ // Inserts key,value pair into the map if the key isn't already in the map.
+ // The value is constructed in-place if the key is not in the map, otherwise
+ // it is not moved.
+ template <typename... Ts>
+ std::pair<iterator, bool> try_emplace(const KeyT &Key, Ts &&...Args) {
+ return try_emplace_impl(Key, std::forward<Ts>(Args)...);
+ }
- typename BaseT::Rep getRep() const { return {Buckets, Used, NumBuckets}; }
+ /// Alternate version of insert() which allows a different, and possibly
+ /// less expensive, key type.
+ /// The DenseMapInfo is responsible for supplying methods
+ /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key
+ /// type used.
+ template <typename LookupKeyT>
+ std::pair<iterator, bool> insert_as(std::pair<KeyT, ValueT> &&KV,
+ const LookupKeyT &Val) {
+ BucketT *TheBucket;
+ if (LookupBucketFor(Val, TheBucket))
+ return {makeIterator(TheBucket), false}; // Already in map.
- void setStorage(void *Storage, unsigned Num) {
- Buckets = static_cast<BucketT *>(Storage);
- Used = llvm::densemap::detail::usedFor(Storage, sizeof(BucketT), Num);
- NumBuckets = Num;
+ // Otherwise, insert the new element.
+ TheBucket = findBucketForInsertion(Val, TheBucket);
+ ::new (&TheBucket->getFirst()) KeyT(std::move(KV.first));
+ ::new (&TheBucket->getSecond()) ValueT(std::move(KV.second));
+ return {makeIterator(TheBucket), true};
}
- void growShared(unsigned MinNumBuckets) {
- unsigned NewNumBuckets = roundUpNumBuckets(MinNumBuckets);
- setStorage(llvm::densemap::detail::growRelocatable(
- Buckets, Used, NumBuckets, NewNumBuckets, sizeof(BucketT),
- llvm::densemap::detail::allocAlign<BucketT>(),
- BaseT::hasher(), /*FreeOld=*/true),
- NewNumBuckets);
+ /// Range insertion of pairs.
+ template <typename InputIt> void insert(InputIt I, InputIt E) {
+ for (; I != E; ++I)
+ insert(*I);
}
- UsedT *getUsed() const { return Used; }
+ /// Inserts range of 'std::pair<KeyT, ValueT>' values into the map.
+ template <typename Range> void insert_range(Range &&R) {
+ insert(adl_begin(R), adl_end(R));
+ }
- unsigned getNumBuckets() const { return NumBuckets; }
+ template <typename V>
+ std::pair<iterator, bool> insert_or_assign(const KeyT &Key, V &&Val) {
+ auto Ret = try_emplace(Key, std::forward<V>(Val));
+ if (!Ret.second)
+ Ret.first->second = std::forward<V>(Val);
+ return Ret;
+ }
- void deallocateBuckets() {
- if (NumBuckets == 0)
- return;
- deallocate_buffer(Buckets,
- llvm::densemap::detail::allocBytes<BucketT>(NumBuckets),
- llvm::densemap::detail::allocAlign<BucketT>());
- Buckets = nullptr;
- Used = nullptr;
- NumBuckets = 0;
+ template <typename V>
+ std::pair<iterator, bool> insert_or_assign(KeyT &&Key, V &&Val) {
+ auto Ret = try_emplace(std::move(Key), std::forward<V>(Val));
+ if (!Ret.second)
+ Ret.first->second = std::forward<V>(Val);
+ return Ret;
}
- bool allocateBuckets(unsigned Num) {
- if (Num == 0) {
- Buckets = nullptr;
- Used = nullptr;
- NumBuckets = 0;
- return false;
- }
- setStorage(allocate_buffer(llvm::densemap::detail::allocBytes<BucketT>(Num),
- llvm::densemap::detail::allocAlign<BucketT>()),
- Num);
- return true;
+ template <typename... Ts>
+ std::pair<iterator, bool> emplace_or_assign(const KeyT &Key, Ts &&...Args) {
+ auto Ret = try_emplace(Key, std::forward<Ts>(Args)...);
+ if (!Ret.second)
+ Ret.first->second = ValueT(std::forward<Ts>(Args)...);
+ return Ret;
}
- // Put the zombie instance in a known good state after a move.
- // deallocateBuckets() already resets to the empty state.
- void kill() { deallocateBuckets(); }
+ template <typename... Ts>
+ std::pair<iterator, bool> emplace_or_assign(KeyT &&Key, Ts &&...Args) {
+ auto Ret = try_emplace(std::move(Key), std::forward<Ts>(Args)...);
+ if (!Ret.second)
+ Ret.first->second = ValueT(std::forward<Ts>(Args)...);
+ return Ret;
+ }
- static unsigned roundUpNumBuckets(unsigned MinNumBuckets) {
- return std::max(64u, MinNumBuckets);
+ void eraseFromFilledBucket(BucketT *TheBucket) {
+ eraseFromFilledBucket(TheBucket, [](BucketT &) {});
}
- bool maybeMoveFast(DenseMap &&Other) {
- swapImpl(Other);
+ bool erase(const KeyT &Val) {
+ BucketT *TheBucket = doFind(Val);
+ if (!TheBucket)
+ return false; // not in map.
+
+ eraseFromFilledBucket(TheBucket);
return true;
}
+ void erase(iterator I) { eraseFromFilledBucket(&*I); }
- // Plan how to shrink the bucket table. Return:
- // - {false, 0} to reuse the existing bucket table
- // - {true, N} to reallocate a bucket table with N entries
- std::pair<bool, unsigned> planShrinkAndClear() const {
- unsigned NewNumBuckets = 0;
- if (NumEntries)
- NewNumBuckets = std::max(64u, 1u << (Log2_32_Ceil(NumEntries) + 1));
- if (NewNumBuckets == NumBuckets)
- return {false, 0}; // Reuse.
- return {true, NewNumBuckets}; // Reallocate.
+ /// Remove entries that match the given predicate. \p Pred is invoked
+ /// with a reference to each live bucket and must not access the map being
+ /// modified. This is the safe replacement for erase-while-iterating.
+ ///
+ /// Returns whether anything was removed. If so, all iterators and references
+ /// into the map are invalidated.
+ template <typename Predicate> bool remove_if(Predicate Pred) {
+ UsedT *U = getUsed();
+ unsigned NumBuckets = getNumBuckets();
+ BucketT *B = getBuckets();
+ bool Removed = false;
+ for (unsigned I = 0; I != NumBuckets; ++I) {
+ if (!llvm::densemap::detail::used(U, I))
+ continue;
+ if (Pred(B[I])) {
+ B[I].getSecond().~ValueT();
+ B[I].getFirst().~KeyT();
+ llvm::densemap::detail::unsetUsed(U, I);
+ decrementNumEntries();
+ Removed = true;
+ }
+ }
+ if (Removed) {
+ incrementEpoch();
+ this->grow(NumBuckets);
+ }
+ return Removed;
}
-};
-
-template <typename KeyT, typename ValueT, unsigned InlineBuckets = 4,
- typename KeyInfoT = DenseMapInfo<KeyT>,
- typename BucketT = llvm::detail::DenseMapPair<KeyT, ValueT>>
-class SmallDenseMap
- : public DenseMapBase<
- SmallDenseMap<KeyT, ValueT, InlineBuckets, KeyInfoT, BucketT>, KeyT,
- ValueT, KeyInfoT, BucketT> {
- friend class DenseMapBase<SmallDenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
- // Lift some types from the dependent base class into this class for
- // simplicity of referring to them.
- using BaseT = DenseMapBase<SmallDenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
- using UsedT = llvm::densemap::detail::UsedT;
+ ValueT &operator[](const KeyT &Key) {
+ return lookupOrInsertIntoBucket(Key).first->second;
+ }
- static_assert(isPowerOf2_64(InlineBuckets),
- "InlineBuckets must be a power of 2.");
+ ValueT &operator[](KeyT &&Key) {
+ return lookupOrInsertIntoBucket(std::move(Key)).first->second;
+ }
- // Number of used words backing the inline buckets (>= 1).
- static constexpr unsigned InlineUsedWords =
- llvm::densemap::detail::usedWords(InlineBuckets);
+ /// Return true if the specified pointer points somewhere into the DenseMap's
+ /// array of buckets (i.e. either to a key or value in the DenseMap).
+ [[nodiscard]] bool isPointerIntoBucketsArray(const void *Ptr) const {
+ return Ptr >= getBuckets() && Ptr < getBucketsEnd();
+ }
- unsigned Small : 1;
- unsigned NumEntries : 31;
+ /// getPointerIntoBucketsArray() - Return an opaque pointer into the buckets
+ /// array. In conjunction with the previous method, this can be used to
+ /// determine whether an insertion caused the DenseMap to reallocate.
+ [[nodiscard]] const void *getPointerIntoBucketsArray() const {
+ return getBuckets();
+ }
- // Inline storage: the bucket array followed by the parallel used words.
- struct InlineRep {
- alignas(BucketT) char Buckets[sizeof(BucketT) * InlineBuckets];
- UsedT Used[InlineUsedWords];
- };
- struct LargeRep {
- BucketT *Buckets;
- UsedT *Used;
- unsigned NumBuckets;
- };
+ void swap(DenseMapBase &RHS) {
+ this->incrementEpoch();
+ RHS.incrementEpoch();
+ Storage.swap(RHS.Storage);
+ }
- // Discriminated by the Small bit.
- union {
- InlineRep Inline;
- LargeRep Large;
- } storage;
+ DenseMapBase() : DenseMapBase(0) {}
- SmallDenseMap(unsigned NumBuckets, typename BaseT::ExactBucketCount) {
- this->initWithExactBucketCount(NumBuckets);
+ /// Create a DenseMap with an optional \p NumElementsToReserve to guarantee
+ /// that this number of elements can be inserted in the map without grow().
+ explicit DenseMapBase(unsigned NumElementsToReserve) {
+ initWithExactBucketCount(
+ getMinBucketToReserveForEntries(NumElementsToReserve));
}
-public:
- explicit SmallDenseMap(unsigned NumElementsToReserve = 0)
- : SmallDenseMap(
- BaseT::getMinBucketToReserveForEntries(NumElementsToReserve),
- typename BaseT::ExactBucketCount{}) {}
-
- SmallDenseMap(const SmallDenseMap &other) : SmallDenseMap() {
+ DenseMapBase(const DenseMapBase &other) : DenseMapBase() {
this->copyFrom(other);
}
- SmallDenseMap(SmallDenseMap &&other) : SmallDenseMap() { this->swap(other); }
+ DenseMapBase(DenseMapBase &&other) : DenseMapBase() { this->swap(other); }
template <typename InputIt>
- SmallDenseMap(const InputIt &I, const InputIt &E)
- : SmallDenseMap(std::distance(I, E)) {
+ DenseMapBase(const InputIt &I, const InputIt &E)
+ : DenseMapBase(std::distance(I, E)) {
this->insert(I, E);
}
template <typename RangeT>
- SmallDenseMap(llvm::from_range_t, const RangeT &Range)
- : SmallDenseMap(adl_begin(Range), adl_end(Range)) {}
+ DenseMapBase(llvm::from_range_t, const RangeT &Range)
+ : DenseMapBase(adl_begin(Range), adl_end(Range)) {}
- SmallDenseMap(std::initializer_list<typename BaseT::value_type> Vals)
- : SmallDenseMap(Vals.begin(), Vals.end()) {}
+ DenseMapBase(std::initializer_list<value_type> Vals)
+ : DenseMapBase(Vals.begin(), Vals.end()) {}
- ~SmallDenseMap() {
+ ~DenseMapBase() {
this->destroyAll();
- deallocateBuckets();
+ Storage.deallocateBuckets();
}
- SmallDenseMap &operator=(const SmallDenseMap &other) {
+ DenseMapBase &operator=(const DenseMapBase &other) {
if (&other != this)
this->copyFrom(other);
return *this;
}
- SmallDenseMap &operator=(SmallDenseMap &&other) {
+ DenseMapBase &operator=(DenseMapBase &&other) {
this->destroyAll();
- deallocateBuckets();
+ Storage.deallocateBuckets();
this->initWithExactBucketCount(0);
this->swap(other);
return *this;
}
-private:
- // Move-construct *Dst from *Src, then destroy *Src. Dst is raw storage.
- static void relocateBucket(BucketT *Dst, BucketT *Src) {
- ::new (&Dst->getFirst()) KeyT(std::move(Src->getFirst()));
- ::new (&Dst->getSecond()) ValueT(std::move(Src->getSecond()));
- Src->getSecond().~ValueT();
- Src->getFirst().~KeyT();
+protected:
+ StorageT Storage;
+
+ struct ExactBucketCount {};
+
+ using Rep = llvm::densemap::detail::StorageRep<BucketT>;
+
+ DenseMapBase(unsigned NumBuckets, ExactBucketCount) {
+ initWithExactBucketCount(NumBuckets);
}
- void swapImpl(SmallDenseMap &RHS) {
- unsigned TmpNumEntries = RHS.NumEntries;
- RHS.NumEntries = NumEntries;
- NumEntries = TmpNumEntries;
+ void initWithExactBucketCount(unsigned NewNumBuckets) {
+ if (Storage.allocateBuckets(NewNumBuckets))
+ initEmpty();
+ else
+ setNumEntries(0);
+ }
- if (Small && RHS.Small) {
- // Both inline: swap the live bucket contents slot by slot, then the used
- // used words. Buckets are raw storage, so a value may only move in one
- // direction when exactly one side is occupied.
- UsedT *LU = getInlineUsed(), *RU = RHS.getInlineUsed();
- BucketT *LB = getInlineBuckets(), *RB = RHS.getInlineBuckets();
- for (unsigned I = 0; I != InlineBuckets; ++I) {
- bool L = llvm::densemap::detail::used(LU, I);
- bool R = llvm::densemap::detail::used(RU, I);
- if (L && R) {
- // Both occupied: exchange through a temporary.
- alignas(BucketT) char Tmp[sizeof(BucketT)];
- BucketT *T = reinterpret_cast<BucketT *>(Tmp);
- relocateBucket(T, &LB[I]);
- relocateBucket(&LB[I], &RB[I]);
- relocateBucket(&RB[I], T);
- } else if (L) {
- relocateBucket(&RB[I], &LB[I]);
- } else if (R) {
- relocateBucket(&LB[I], &RB[I]);
- }
- }
- for (unsigned W = 0; W != InlineUsedWords; ++W)
- std::swap(LU[W], RU[W]);
+ void destroyAll() {
+ // No need to iterate through the buckets if the bucket is trivially
+ // destructible.
+ if constexpr (std::is_trivially_destructible_v<BucketT>)
return;
- }
- if (!Small && !RHS.Small) {
- std::swap(storage.Large, RHS.storage.Large);
+
+ if (getNumBuckets() == 0) // Nothing to do.
+ return;
+
+ BucketT *B = getBuckets();
+ const UsedT *U = getUsed();
+ const unsigned E = getNumBuckets();
+ llvm::densemap::detail::forEachUsed(U, E, [&](unsigned I) {
+ B[I].getSecond().~ValueT();
+ B[I].getFirst().~KeyT();
+ });
+ }
+
+ void initEmpty() {
+ setNumEntries(0);
+
+ assert((getNumBuckets() & (getNumBuckets() - 1)) == 0 &&
+ "# initial buckets must be a power of two!");
+ if (getNumBuckets())
+ llvm::densemap::detail::clearUsed(getUsed(), getNumBuckets());
+ }
+
+ /// Returns the number of buckets to allocate to ensure that the DenseMap can
+ /// accommodate \p NumEntries without need to grow().
+ unsigned getMinBucketToReserveForEntries(unsigned NumEntries) {
+ // Ensure that "NumEntries * 4 < NumBuckets * 3"
+ if (NumEntries == 0)
+ return 0;
+ // +1 is required because of the strict inequality.
+ // For example, if NumEntries is 48, we need to return 128.
+ return NextPowerOf2(NumEntries * 4 / 3 + 1);
+ }
+
+ static constexpr llvm::densemap::detail::BucketHasher hasher() {
+ return llvm::densemap::detail::hasherFor<KeyT, KeyInfoT>();
+ }
+
+ // Move key/value from Other to *this.
+ // Other is left in a valid but empty state.
+ LLVM_ATTRIBUTE_NOINLINE void moveFrom(DenseMapBase &Other) {
+ assert(getNumEntries() == 0 && "moveFrom requires an empty destination");
+ BucketT *OtherB = Other.getBuckets();
+ UsedT *OtherU = Other.getUsed();
+ const unsigned E = Other.getNumBuckets();
+ UsedT *U = getUsed();
+ BucketT *B = getBuckets();
+ const unsigned Mask = getNumBuckets() - 1;
+ llvm::densemap::detail::forEachUsed(OtherU, E, [&](unsigned I) {
+ // Find the first empty slot on this key's probe chain; there is no equal
+ // key in the destination, so nothing to compare against.
+ unsigned BucketNo = KeyInfoT::getHashValue(OtherB[I].getFirst()) & Mask;
+ while (llvm::densemap::detail::used(U, BucketNo))
+ BucketNo = (BucketNo + 1) & Mask;
+ BucketT *DestBucket = B + BucketNo;
+ ::new (&DestBucket->getFirst()) KeyT(std::move(OtherB[I].getFirst()));
+ ::new (&DestBucket->getSecond()) ValueT(std::move(OtherB[I].getSecond()));
+ llvm::densemap::detail::setUsed(U, BucketNo);
+
+ // Free the moved-out key/value.
+ OtherB[I].getSecond().~ValueT();
+ OtherB[I].getFirst().~KeyT();
+ });
+ setNumEntries(Other.getNumEntries());
+ Other.Storage.kill();
+ }
+
+ LLVM_ATTRIBUTE_NOINLINE void copyFrom(const DenseMapBase &other) {
+ this->destroyAll();
+ Storage.deallocateBuckets();
+ setNumEntries(0);
+ if (!Storage.allocateBuckets(other.getNumBuckets())) {
+ // The bucket list is empty. No work to do.
return;
}
- SmallDenseMap &SmallSide = Small ? *this : RHS;
- SmallDenseMap &LargeSide = Small ? RHS : *this;
+ assert(&other != this);
+ assert(getNumBuckets() == other.getNumBuckets());
- // Stash the large rep, then move the small side's inline contents into the
- // large side (which becomes inline), and finally install the rep on the
- // small side (which becomes large).
- LargeRep TmpRep = LargeSide.storage.Large;
- LargeSide.Small = true;
- {
- UsedT *SU = SmallSide.getInlineUsed(), *LU = LargeSide.getInlineUsed();
- BucketT *SB = SmallSide.getInlineBuckets(),
- *LB = LargeSide.getInlineBuckets();
- for (unsigned I = 0; I != InlineBuckets; ++I)
- if (llvm::densemap::detail::used(SU, I))
- relocateBucket(&LB[I], &SB[I]);
- for (unsigned W = 0; W != InlineUsedWords; ++W)
- LU[W] = SU[W];
+ setNumEntries(other.getNumEntries());
+
+ BucketT *Buckets = getBuckets();
+ const BucketT *OtherBuckets = other.getBuckets();
+ const unsigned NumBuckets = getNumBuckets();
+ UsedT *U = getUsed();
+ const UsedT *OtherU = other.getUsed();
+ std::memcpy(U, OtherU,
+ llvm::densemap::detail::usedWords(NumBuckets) * sizeof(UsedT));
+ if constexpr (densemap::detail::isRelocatableBucket<BucketT>) {
+ memcpy(reinterpret_cast<void *>(Buckets), OtherBuckets,
+ NumBuckets * sizeof(BucketT));
+ } else {
+ llvm::densemap::detail::forEachUsed(U, NumBuckets, [&](unsigned I) {
+ ::new (&Buckets[I].getFirst()) KeyT(OtherBuckets[I].getFirst());
+ ::new (&Buckets[I].getSecond()) ValueT(OtherBuckets[I].getSecond());
+ });
}
- SmallSide.Small = false;
- SmallSide.storage.Large = TmpRep;
}
- unsigned getNumEntries() const { return NumEntries; }
+private:
+ // ValueHandleBase caches pointers into the bucket array, so it needs the
+ // callback erase below to fix them up as entries shift. It is the only
+ // intended caller; do not add new ones.
+ friend class ValueHandleBase;
+
+ /// Erase the entry at \p TheBucket and close the resulting hole via Knuth
+ /// TAOCP 6.4 Algorithm R. For callers that cache pointers into the bucket
+ /// array, call \p OnMoved per shifted bucket.
+ template <typename OnMovedT>
+ LLVM_ATTRIBUTE_NOINLINE void eraseFromFilledBucket(BucketT *TheBucket,
+ OnMovedT &&OnMoved) {
+ incrementEpoch();
+ TheBucket->getSecond().~ValueT();
+ TheBucket->getFirst().~KeyT();
+ decrementNumEntries();
+
+ BucketT *BucketsPtr = getBuckets();
+ UsedT *U = getUsed();
+ const unsigned Mask = getNumBuckets() - 1;
+ unsigned I = TheBucket - BucketsPtr;
+ unsigned J = I;
+ while (true) {
+ J = (J + 1) & Mask;
+ BucketT &BJ = BucketsPtr[J];
+ if (!llvm::densemap::detail::used(U, J))
+ break;
+ auto Ideal = KeyInfoT::getHashValue(BJ.getFirst());
+ // If the hole (I) lies on the linear-probe chain from the home bucket
+ // (Ideal) to J, shift J into the hole and make J the new hole.
+ if (((I - Ideal) & Mask) < ((J - Ideal) & Mask)) {
+ BucketT &BI = BucketsPtr[I];
+ ::new (&BI.getFirst()) KeyT(std::move(BJ.getFirst()));
+ ::new (&BI.getSecond()) ValueT(std::move(BJ.getSecond()));
+ BJ.getSecond().~ValueT();
+ BJ.getFirst().~KeyT();
+ OnMoved(BI);
+ I = J;
+ }
+ }
+ llvm::densemap::detail::unsetUsed(U, I);
+ }
+
+ /// Erase \p Val and close the resulting hole by potentially shifting other
+ /// entries into it. For callers that cache pointers into the bucket array,
+ /// call \p OnMoved per shifted bucket.
+ template <typename OnMovedT> bool erase(const KeyT &Val, OnMovedT &&OnMoved) {
+ BucketT *TheBucket = doFind(Val);
+ if (!TheBucket)
+ return false;
+ eraseFromFilledBucket(TheBucket, std::forward<OnMovedT>(OnMoved));
+ return true;
+ }
+
+ template <typename KeyArgT, typename... Ts>
+ std::pair<BucketT *, bool> lookupOrInsertIntoBucket(KeyArgT &&Key,
+ Ts &&...Args) {
+ BucketT *TheBucket = nullptr;
+ if (LookupBucketFor(Key, TheBucket))
+ return {TheBucket, false}; // Already in the map.
- void setNumEntries(unsigned Num) {
- // NumEntries is hardcoded to be 31 bits wide.
- assert(Num < (1U << 31) && "Cannot support more than 1<<31 entries");
- NumEntries = Num;
+ // Otherwise, insert the new element.
+ TheBucket = findBucketForInsertion(Key, TheBucket);
+ ::new (&TheBucket->getFirst()) KeyT(std::forward<KeyArgT>(Key));
+ ::new (&TheBucket->getSecond()) ValueT(std::forward<Ts>(Args)...);
+ return {TheBucket, true};
}
- const BucketT *getInlineBuckets() const {
- assert(Small);
- // Note that this cast does not violate aliasing rules as we assert that
- // the memory's dynamic type is the small, inline bucket buffer, and the
- // 'storage' is a POD containing a char buffer.
- return reinterpret_cast<const BucketT *>(storage.Inline.Buckets);
+ template <typename KeyArgT, typename... Ts>
+ std::pair<iterator, bool> try_emplace_impl(KeyArgT &&Key, Ts &&...Args) {
+ auto [Bucket, Inserted] = lookupOrInsertIntoBucket(
+ std::forward<KeyArgT>(Key), std::forward<Ts>(Args)...);
+ return {makeIterator(Bucket), Inserted};
}
- BucketT *getInlineBuckets() {
- assert(Small);
- return reinterpret_cast<BucketT *>(storage.Inline.Buckets);
+ iterator makeIterator(BucketT *TheBucket) {
+ return iterator::makeIterator(TheBucket, getBuckets(), getUsed(),
+ getNumBuckets(), *this);
}
- const UsedT *getInlineUsed() const {
- assert(Small);
- return storage.Inline.Used;
+ const_iterator makeConstIterator(const BucketT *TheBucket) const {
+ return const_iterator::makeIterator(TheBucket, getBuckets(), getUsed(),
+ getNumBuckets(), *this);
}
- UsedT *getInlineUsed() {
- assert(Small);
- return storage.Inline.Used;
- }
+ unsigned getNumEntries() const { return Storage.getNumEntries(); }
- const BucketT *getBuckets() const {
- return Small ? getInlineBuckets() : storage.Large.Buckets;
- }
+ void setNumEntries(unsigned Num) { Storage.setNumEntries(Num); }
- typename BaseT::Rep getRep() const {
- if (Small)
- return {getInlineBuckets(), getInlineUsed(), InlineBuckets};
- return {storage.Large.Buckets, storage.Large.Used,
- storage.Large.NumBuckets};
- }
+ void incrementNumEntries() { setNumEntries(getNumEntries() + 1); }
- BucketT *getBuckets() {
- return const_cast<BucketT *>(
- const_cast<const SmallDenseMap *>(this)->getBuckets());
+ void decrementNumEntries() { setNumEntries(getNumEntries() - 1); }
+
+ const BucketT *getBuckets() const { return Storage.getBuckets(); }
+
+ BucketT *getBuckets() { return Storage.getBuckets(); }
+
+ Rep getRep() const { return Storage.getRep(); }
+
+ const UsedT *getUsed() const { return Storage.getUsed(); }
+
+ UsedT *getUsed() { return Storage.getUsed(); }
+
+ unsigned getNumBuckets() const { return Storage.getNumBuckets(); }
+
+ BucketT *getBucketsEnd() { return getBuckets() + getNumBuckets(); }
+
+ const BucketT *getBucketsEnd() const {
+ return getBuckets() + getNumBuckets();
}
- const UsedT *getUsed() const {
- return Small ? getInlineUsed() : storage.Large.Used;
+ LLVM_ATTRIBUTE_NOINLINE void grow(unsigned MinNumBuckets) {
+ assert((MinNumBuckets == 0 || isPowerOf2_32(MinNumBuckets)) &&
+ "bucket count must be zero or a power of two");
+ if constexpr (llvm::densemap::detail::isRelocatableBucket<BucketT>) {
+ Storage.growShared(MinNumBuckets, hasher());
+ } else {
+ unsigned NumBuckets = StorageT::roundUpNumBuckets(MinNumBuckets);
+ DenseMapBase Tmp(NumBuckets, ExactBucketCount{});
+ Tmp.moveFrom(*this);
+ if (Storage.maybeMoveFast(std::move(Tmp.Storage)))
+ return;
+ initWithExactBucketCount(NumBuckets);
+ moveFrom(Tmp);
+ }
}
- UsedT *getUsed() {
- return const_cast<UsedT *>(
- const_cast<const SmallDenseMap *>(this)->getUsed());
+ template <typename LookupKeyT>
+ BucketT *findBucketForInsertion(const LookupKeyT &Lookup,
+ BucketT *TheBucket) {
+ incrementEpoch();
+
+ // Grow the table if the load factor would exceed 3/4 after insertion.
+ // Linear probing with gap-closing deletion (Knuth Algorithm R) keeps
+ // every chain compact and bounded by the table's empty-bucket count,
+ // so no tombstone-driven resize is needed.
+ unsigned NewNumEntries = getNumEntries() + 1;
+ unsigned NumBuckets = getNumBuckets();
+ if (LLVM_UNLIKELY(NewNumEntries * 4 >= NumBuckets * 3)) {
+ this->grow(NumBuckets * 2);
+ LookupBucketFor(Lookup, TheBucket);
+ }
+ assert(TheBucket);
+
+ // Mark used. The caller will placement-construct the raw key/value.
+ llvm::densemap::detail::setUsed(getUsed(), TheBucket - getBuckets());
+
+ // Only update the state after we've grown our bucket space appropriately
+ // so that when growing buckets we have self-consistent entry count.
+ incrementNumEntries();
+ return TheBucket;
}
- unsigned getNumBuckets() const {
- return Small ? InlineBuckets : storage.Large.NumBuckets;
+ template <typename LookupKeyT>
+ const BucketT *doFind(const LookupKeyT &Val) const {
+ if (empty())
+ return nullptr;
+ auto [BucketsPtr, U, NumBuckets] = getRep();
+
+ const unsigned Mask = NumBuckets - 1;
+ unsigned BucketNo = KeyInfoT::getHashValue(Val) & Mask;
+ while (true) {
+ // An empty bucket terminates the probe: the key isn't in the map.
+ if (LLVM_LIKELY(!llvm::densemap::detail::used(U, BucketNo)))
+ return nullptr;
+ const BucketT *Bucket = BucketsPtr + BucketNo;
+ if (LLVM_LIKELY(KeyInfoT::isEqual(Val, Bucket->getFirst())))
+ return Bucket;
+
+ // Hash collision: continue linear probing.
+ BucketNo = (BucketNo + 1) & Mask;
+ }
}
- void setLarge(void *Storage, unsigned NumBuckets) {
- Small = false;
- storage.Large = {
- static_cast<BucketT *>(Storage),
- llvm::densemap::detail::usedFor(Storage, sizeof(BucketT), NumBuckets),
- NumBuckets};
+ template <typename LookupKeyT> BucketT *doFind(const LookupKeyT &Val) {
+ return const_cast<BucketT *>(
+ static_cast<const DenseMapBase *>(this)->doFind(Val));
}
- void growShared(unsigned MinNumBuckets) {
- unsigned NewNumBuckets = roundUpNumBuckets(MinNumBuckets);
- // remove_if asks for the count it already has: rehash in place.
- if (Small && NewNumBuckets <= InlineBuckets) {
- InlineRep Old = storage.Inline;
- llvm::densemap::detail::clearUsed(getInlineUsed(), InlineBuckets);
- llvm::densemap::detail::rehashRelocatable(
- getInlineBuckets(), getInlineUsed(), InlineBuckets, Old.Buckets,
- Old.Used, InlineBuckets, sizeof(BucketT), BaseT::hasher());
- return;
+ /// Lookup the appropriate bucket for Val, returning it in FoundBucket. If the
+ /// bucket contains the key and a value, this returns true, otherwise it
+ /// returns a bucket with an empty marker and returns false.
+ template <typename LookupKeyT>
+ bool LookupBucketFor(const LookupKeyT &Val, BucketT *&FoundBucket) {
+ auto [CBuckets, U, NumBuckets] = getRep();
+ if (NumBuckets == 0) {
+ FoundBucket = nullptr;
+ return false;
}
- void *Storage = llvm::densemap::detail::growRelocatable(
- getBuckets(), getUsed(), getNumBuckets(), NewNumBuckets,
- sizeof(BucketT), llvm::densemap::detail::allocAlign<BucketT>(),
- BaseT::hasher(), /*FreeOld=*/!Small);
- setLarge(Storage, NewNumBuckets);
- }
+ // getRep() yields const pointers; this object is non-const, so recovering
+ // a mutable bucket pointer is safe (mirrors the non-const getBuckets()).
+ BucketT *BucketsPtr = const_cast<BucketT *>(CBuckets);
- void deallocateBuckets() {
- // Fast path in case storage.Large.NumBuckets == 0, just like destroyAll.
- // This path is used to destruct zombie instances after moves.
- if (Small || storage.Large.NumBuckets == 0)
- return;
+ const unsigned Mask = NumBuckets - 1;
+ unsigned BucketNo = KeyInfoT::getHashValue(Val) & Mask;
+ while (true) {
+ BucketT *ThisBucket = BucketsPtr + BucketNo;
+ // If we found an empty bucket, the key doesn't exist in the set.
+ // Return it as the insertion point.
+ if (LLVM_LIKELY(!llvm::densemap::detail::used(U, BucketNo))) {
+ FoundBucket = ThisBucket;
+ return false;
+ }
- deallocate_buffer(
- storage.Large.Buckets,
- llvm::densemap::detail::allocBytes<BucketT>(storage.Large.NumBuckets),
- llvm::densemap::detail::allocAlign<BucketT>());
- storage.Large.NumBuckets = 0;
- }
+ // Found Val's bucket? If so, return it.
+ if (LLVM_LIKELY(KeyInfoT::isEqual(Val, ThisBucket->getFirst()))) {
+ FoundBucket = ThisBucket;
+ return true;
+ }
- bool allocateBuckets(unsigned Num) {
- if (Num <= InlineBuckets) {
- Small = true;
- return true;
+ // Hash collision: continue linear probing.
+ BucketNo = (BucketNo + 1) & Mask;
}
- setLarge(allocate_buffer(llvm::densemap::detail::allocBytes<BucketT>(Num),
- llvm::densemap::detail::allocAlign<BucketT>()),
- Num);
- return true;
}
- // Put the zombie instance in a known good state after a move.
- void kill() {
- deallocateBuckets();
- Small = false;
- storage.Large = LargeRep{nullptr, nullptr, 0};
+public:
+ /// Return the approximate size (in bytes) of the actual map.
+ /// This is just the raw memory used by DenseMap.
+ /// If entries are pointers to objects, the size of the referenced objects
+ /// are not included.
+ [[nodiscard]] size_t getMemorySize() const {
+ return llvm::densemap::detail::allocBytes<BucketT>(getNumBuckets());
}
+};
- static unsigned roundUpNumBuckets(unsigned MinNumBuckets) {
- if (MinNumBuckets <= InlineBuckets)
- return InlineBuckets;
- return std::max(64u, MinNumBuckets);
- }
+/// Equality comparison for DenseMap.
+///
+/// Iterates over elements of LHS confirming that each (key, value) pair in LHS
+/// is also in RHS, and that no additional pairs are in RHS.
+/// Equivalent to N calls to RHS.find and N value comparisons. Amortized
+/// complexity is linear, worst case is O(N^2) (if every hash collides).
+template <typename Storage1T, typename Storage2T, typename KeyT,
+ typename ValueT, typename KeyInfoT, typename BucketT>
+[[nodiscard]] bool operator==(
+ const DenseMapBase<Storage1T, KeyT, ValueT, KeyInfoT, BucketT> &LHS,
+ const DenseMapBase<Storage2T, KeyT, ValueT, KeyInfoT, BucketT> &RHS) {
+ if (LHS.size() != RHS.size())
+ return false;
- bool maybeMoveFast(SmallDenseMap &&Other) {
- if (Other.Small)
+ for (auto &KV : LHS) {
+ auto I = RHS.find(KV.first);
+ if (I == RHS.end() || I->second != KV.second)
return false;
-
- Small = false;
- NumEntries = Other.NumEntries;
- storage.Large = Other.storage.Large;
- Other.storage.Large.NumBuckets = 0;
- return true;
}
- // Plan how to shrink the bucket table. Return:
- // - {false, 0} to reuse the existing bucket table
- // - {true, N} to reallocate a bucket table with N entries
- std::pair<bool, unsigned> planShrinkAndClear() const {
- unsigned NewNumBuckets = 0;
- if (!this->empty()) {
- NewNumBuckets = 1u << (Log2_32_Ceil(this->size()) + 1);
- if (NewNumBuckets > InlineBuckets)
- NewNumBuckets = std::max(64u, NewNumBuckets);
- }
- bool Reuse = Small ? NewNumBuckets <= InlineBuckets
- : NewNumBuckets == storage.Large.NumBuckets;
- if (Reuse)
- return {false, 0}; // Reuse.
- return {true, NewNumBuckets}; // Reallocate.
- }
+ return true;
+}
+
+/// Inequality comparison for DenseMap.
+///
+/// Equivalent to !(LHS == RHS). See operator== for performance notes.
+template <typename Storage1T, typename Storage2T, typename KeyT,
+ typename ValueT, typename KeyInfoT, typename BucketT>
+[[nodiscard]] bool operator!=(
+ const DenseMapBase<Storage1T, KeyT, ValueT, KeyInfoT, BucketT> &LHS,
+ const DenseMapBase<Storage2T, KeyT, ValueT, KeyInfoT, BucketT> &RHS) {
+ return !(LHS == RHS);
+}
+
+template <typename KeyT, typename ValueT,
+ typename KeyInfoT = DenseMapInfo<KeyT>,
+ typename BucketT = llvm::detail::DenseMapPair<KeyT, ValueT>>
+class DenseMap : public DenseMapBase<densemap::detail::DenseMapStorage<BucketT>,
+ KeyT, ValueT, KeyInfoT, BucketT> {
+ using BaseT = DenseMapBase<densemap::detail::DenseMapStorage<BucketT>, KeyT,
+ ValueT, KeyInfoT, BucketT>;
+
+public:
+ using BaseT::BaseT;
+};
+
+template <typename KeyT, typename ValueT, unsigned InlineBuckets = 4,
+ typename KeyInfoT = DenseMapInfo<KeyT>,
+ typename BucketT = llvm::detail::DenseMapPair<KeyT, ValueT>>
+class SmallDenseMap
+ : public DenseMapBase<
+ densemap::detail::SmallDenseMapStorage<BucketT, InlineBuckets>, KeyT,
+ ValueT, KeyInfoT, BucketT> {
+ using BaseT = DenseMapBase<
+ densemap::detail::SmallDenseMapStorage<BucketT, InlineBuckets>, KeyT,
+ ValueT, KeyInfoT, BucketT>;
+
+public:
+ using BaseT::BaseT;
};
template <typename KeyT, typename ValueT, typename KeyInfoT, typename Bucket,
More information about the llvm-commits
mailing list