[llvm] [IR] Reduce memory usage in metadata use tracking (NFC) (PR #221866)
Kazu Hirata via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 13 12:11:43 PDT 2026
https://github.com/kazutakahirata updated https://github.com/llvm/llvm-project/pull/221866
>From 8ee03c7132753c77b85a99da4735fc6d0cdfa83d Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Sun, 13 Sep 2026 11:56:29 -0700
Subject: [PATCH 1/2] [IR] Reduce memory usage in metadata use tracking (NFC)
This patch reduces the memory footprint of metadata use tracking.
Currently, ReplaceableUses uses:
SmallDenseMap<void *, std::pair<OwnerTy, uint64_t>, 4>
where the uint64_t records insertion order. The problem is that
once it exceeds 2 elements, it allocates 64 buckets (~1.5 KB). In large
compilations with debug info, thousands of metadata nodes have 3 or 4
uses, leading to significant overhead.
With this patch, uses are stored as:
struct UseEntry {
void *Ref;
OwnerTy Owner;
};
SmallVector<UseEntry, 4> UseMap;
// In addRef:
UseMap.push_back({Ref, Owner});
Unlike SmallDenseMap, SmallVector stores all 4 elements inline without
heap allocation. It also preserves insertion order, eliminating the
need to sort uses in several places.
To support this representation efficiently:
- dropRef places a tombstone (Ref = nullptr) in UseMap.
- addRef appends entries with UseMap.push_back(). When UseMap reaches
capacity, we compact UseMap in place if there are at least as many
tombstones as live elements; otherwise, we double capacity.
- MetadataUseMap, a central DenseMap in LLVMContextImpl, maps void *Ref
to its index in UseMap for O(1) lookups in dropRef and moveRef. We
use this DenseMap only after UseMap reaches 5 or more elements;
otherwise, we scan UseMap linearly without querying the map.
- During LLVMContextImpl teardown, MetadataUseMap is cleared in one batch
so subsequent dropRef calls skip map lookups when the map is empty.
- ReplaceableUses accesses LLVMContext for MetadataUseMap via a 1-byte
Kind discriminator placed in existing alignment padding (0 bytes added).
Benchmark results:
Base Test
File Max RSS Max RSS Delta Wall Time
-----------------------------------------------------------------
SLPVectorizer.ii 1300 MB 1159 MB -10.85% 43.76s -> 43.39s
SemaOpenMP.ii 1281 MB 1112 MB -13.16% 34.29s -> 33.88s
X86ISelLowering.ii 831 MB 792 MB -4.65% 26.24s -> 26.10s
-----------------------------------------------------------------
Total 3412 MB 3064 MB -10.21% 104.29s -> 103.37s
Geometric Mean -9.62% -0.86%
- Compilation flags: -O3 -g -fdebug-info-for-profiling -flto=thin
- Max RSS: average of 3 runs after 1 warmup run.
- Wall Time: average of 10 runs after 1 warmup run.
Assisted-by: Antigravity
---
llvm/include/llvm/IR/Metadata.h | 45 +++++-
llvm/lib/IR/LLVMContextImpl.cpp | 4 +
llvm/lib/IR/LLVMContextImpl.h | 5 +
llvm/lib/IR/Metadata.cpp | 269 +++++++++++++++++++++++---------
4 files changed, 245 insertions(+), 78 deletions(-)
diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h
index c677f9cc61769..b6d006d20631e 100644
--- a/llvm/include/llvm/IR/Metadata.h
+++ b/llvm/include/llvm/IR/Metadata.h
@@ -394,11 +394,35 @@ class ReplaceableUses {
public:
using OwnerTy = MetadataTracking::OwnerTy;
+ enum Kind : uint8_t { ValueAsMetadataKind, WithContextKind };
+
+ struct UseEntry {
+ void *Ref = nullptr;
+ OwnerTy Owner = nullptr;
+ };
+
private:
- uint64_t NextIndex = 0;
- SmallDenseMap<void *, std::pair<OwnerTy, uint64_t>, 4> UseMap;
+ // Stores uses in insertion order, with dropped uses tombstoned (Ref =
+ // nullptr) to avoid shifting elements. Lookups use a two-tier hybrid
+ // strategy:
+ // - Small Mode (<= 4 uses): Scans linearly without auxiliary data structures.
+ // - Large Mode (>= 5 uses): Registers reverse lookups (Ref -> index) in
+ // LLVMContextImpl::MetadataUseMap to maintain O(1) dropRef and moveRef.
+ SmallVector<UseEntry, 4> UseMap;
+
+ // The number of tombstones in UseMap.
+ unsigned NumDead = 0;
+
+ // True if we have switched to large mode. Once switched, we stay in
+ // large mode until UseMap becomes empty.
+ bool IsLarge = false;
+
+ Kind K;
+
+ template <bool IsLargeMode> void compact();
protected:
+ ReplaceableUses(Kind K) : K(K) {}
~ReplaceableUses() {
assert(UseMap.empty() && "Cannot destroy in-use replaceable metadata");
}
@@ -406,6 +430,9 @@ class ReplaceableUses {
public:
ReplaceableUses &operator=(const ReplaceableUses &) = delete;
+ Kind getKind() const { return K; }
+ LLVMContext &getContext() const;
+
/// Replace all uses of this with MD.
///
/// Replace all uses of this with \c MD, which is allowed to be null.
@@ -424,12 +451,15 @@ class ReplaceableUses {
/// is resolved.
LLVM_ABI void resolveAllUses(bool ResolveUsers = true);
- unsigned getNumUses() const { return UseMap.size(); }
+ unsigned getNumUses() const { return UseMap.size() - NumDead; }
private:
+ SmallVector<UseEntry, 8> getLiveUses() const;
+ bool hasRef(void *Ref) const;
void addRef(void *Ref, OwnerTy Owner);
void dropRef(void *Ref);
void moveRef(void *Ref, void *New, const Metadata &MD);
+ void clear();
/// Lazily construct RAUW support on MD.
///
@@ -453,7 +483,11 @@ class ReplaceableUsesWithContext : public ReplaceableUses {
public:
explicit ReplaceableUsesWithContext(LLVMContext &Context)
- : Context(Context) {}
+ : ReplaceableUses(WithContextKind), Context(Context) {}
+
+ static bool classof(const ReplaceableUses *R) {
+ return R->getKind() == WithContextKind;
+ }
LLVMContext &getContext() const { return Context; }
};
@@ -478,7 +512,8 @@ class ValueAsMetadata : public Metadata, ReplaceableUses {
}
protected:
- ValueAsMetadata(unsigned ID, Value *V) : Metadata(ID, Uniqued), V(V) {
+ ValueAsMetadata(unsigned ID, Value *V)
+ : Metadata(ID, Uniqued), ReplaceableUses(ValueAsMetadataKind), V(V) {
assert(V && "Expected valid value");
}
diff --git a/llvm/lib/IR/LLVMContextImpl.cpp b/llvm/lib/IR/LLVMContextImpl.cpp
index 5af5a477cf69a..090fb87403ae6 100644
--- a/llvm/lib/IR/LLVMContextImpl.cpp
+++ b/llvm/lib/IR/LLVMContextImpl.cpp
@@ -68,6 +68,10 @@ LLVMContextImpl::~LLVMContextImpl() {
"Values with metadata have been leaked");
#endif
+ // Clear MetadataUseMap in one batch; subsequent dropRef calls will skip map
+ // lookups when the map is empty.
+ MetadataUseMap.clear();
+
// Drop references for MDNodes. Do this before Values get deleted to avoid
// unnecessary RAUW when nodes are still unresolved.
for (auto *I : DistinctMDNodes)
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index 49a8a803732a5..848678a38112f 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -1636,6 +1636,11 @@ class LLVMContextImpl {
DenseMap<Value *, ValueAsMetadata *> ValuesAsMetadata;
DenseMap<Metadata *, MetadataAsValue *> MetadataAsValues;
DenseSet<DIArgList *, DIArgListInfo> DIArgLists;
+ // Maps a tracked metadata reference pointer (void *Ref) to its current index
+ // in ReplaceableUses::UseMap. Only populated for replaceable metadata
+ // instances in Large Mode (IsLarge == true). Cleared in one batch at
+ // teardown so subsequent dropRef calls skip map lookups.
+ DenseMap<void *, unsigned> MetadataUseMap;
uint32_t NextMetadataPrintID = 0;
diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index fc03bf4a87eb1..0c1f21a50d78c 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -251,86 +251,220 @@ bool MetadataTracking::isReplaceable(const Metadata &MD) {
return ReplaceableUses::isReplaceable(MD);
}
+LLVMContext &ReplaceableUses::getContext() const {
+ if (auto *WithContext = dyn_cast<ReplaceableUsesWithContext>(this))
+ return WithContext->getContext();
+ return static_cast<const ValueAsMetadata *>(this)->getContext();
+}
+
+SmallVector<ReplaceableUses::UseEntry, 8>
+ReplaceableUses::getLiveUses() const {
+ SmallVector<UseEntry, 8> LiveUses;
+ LiveUses.reserve(getNumUses());
+ for (const auto &Entry : UseMap) {
+ if (Entry.Ref)
+ LiveUses.push_back(Entry);
+ }
+ return LiveUses;
+}
+
SmallVector<Metadata *> ReplaceableUses::getAllArgListUsers() {
- SmallVector<std::pair<OwnerTy, uint64_t> *> MDUsersWithID;
- for (auto Pair : UseMap) {
- OwnerTy Owner = Pair.second.first;
+ SmallVector<Metadata *> MDUsers;
+ for (const auto &[Ref, Owner] : UseMap) {
+ if (!Ref)
+ continue;
if (Owner.isNull())
continue;
if (!isa<Metadata *>(Owner))
continue;
Metadata *OwnerMD = cast<Metadata *>(Owner);
if (OwnerMD->getMetadataID() == Metadata::DIArgListKind)
- MDUsersWithID.push_back(&UseMap[Pair.first]);
+ MDUsers.push_back(OwnerMD);
}
- llvm::sort(MDUsersWithID, [](auto UserA, auto UserB) {
- return UserA->second < UserB->second;
- });
- SmallVector<Metadata *> MDUsers;
- for (auto *UserWithID : MDUsersWithID)
- MDUsers.push_back(cast<Metadata *>(UserWithID->first));
return MDUsers;
}
SmallVector<DbgVariableRecord *>
ReplaceableUses::getAllDbgVariableRecordUsers() {
- SmallVector<std::pair<OwnerTy, uint64_t> *> DVRUsersWithID;
- for (auto Pair : UseMap) {
- OwnerTy Owner = Pair.second.first;
- if (Owner.isNull())
- continue;
- if (!isa<DebugValueUser *>(Owner))
- continue;
- DVRUsersWithID.push_back(&UseMap[Pair.first]);
- }
// Order DbgVariableRecord users in reverse-creation order. Normal dbg.value
// users of MetadataAsValues are ordered by their UseList, i.e. reverse order
// of when they were added: we need to replicate that here. The structure of
// debug-info output depends on the ordering of intrinsics, thus we need
// to keep them consistent for comparisons sake.
- llvm::sort(DVRUsersWithID, [](auto UserA, auto UserB) {
- return UserA->second > UserB->second;
- });
SmallVector<DbgVariableRecord *> DVRUsers;
- for (auto UserWithID : DVRUsersWithID)
- DVRUsers.push_back(cast<DebugValueUser *>(UserWithID->first)->getUser());
+ for (const auto &[Ref, Owner] : reverse(UseMap)) {
+ if (!Ref)
+ continue;
+ if (Owner.isNull())
+ continue;
+ if (!isa<DebugValueUser *>(Owner))
+ continue;
+ DVRUsers.push_back(cast<DebugValueUser *>(Owner)->getUser());
+ }
return DVRUsers;
}
+bool ReplaceableUses::hasRef(void *Ref) const {
+ if (LLVM_LIKELY(!IsLarge)) {
+ for (const auto &Entry : UseMap) {
+ if (Entry.Ref == Ref)
+ return true;
+ }
+ return false;
+ }
+ return getContext().pImpl->MetadataUseMap.contains(Ref);
+}
+
+// Filters out tombstones in place while preserving insertion order. In Large
+// Mode, updates MetadataUseMap only for elements whose indices actually shift.
+template <bool IsLargeMode> void ReplaceableUses::compact() {
+ DenseMap<void *, unsigned> *Map = nullptr;
+ if constexpr (IsLargeMode)
+ Map = &getContext().pImpl->MetadataUseMap;
+
+ unsigned WriteIdx = 0;
+ for (unsigned ReadIdx = 0, E = UseMap.size(); ReadIdx != E; ++ReadIdx) {
+ if (UseMap[ReadIdx].Ref != nullptr) {
+ if (WriteIdx != ReadIdx) {
+ UseMap[WriteIdx] = UseMap[ReadIdx];
+ if constexpr (IsLargeMode)
+ (*Map)[UseMap[WriteIdx].Ref] = WriteIdx;
+ }
+ ++WriteIdx;
+ }
+ }
+ UseMap.truncate(WriteIdx);
+ NumDead = 0;
+}
+
void ReplaceableUses::addRef(void *Ref, OwnerTy Owner) {
- bool WasInserted =
- UseMap.insert(std::make_pair(Ref, std::make_pair(Owner, NextIndex)))
- .second;
- (void)WasInserted;
- assert(WasInserted && "Expected to add a reference");
+ assert(Ref && "Expected live reference");
+ assert(!hasRef(Ref) && "Reference already tracked");
- ++NextIndex;
- assert(NextIndex != 0 && "Unexpected overflow");
+ if (!IsLarge) {
+ if (LLVM_LIKELY(UseMap.size() < 4)) {
+ UseMap.push_back({Ref, Owner});
+ return;
+ }
+
+ if (NumDead) {
+ compact</*IsLargeMode=*/false>();
+ UseMap.push_back({Ref, Owner});
+ return;
+ }
+
+ // Transition to large mode on the 5th live use.
+ IsLarge = true;
+ UseMap.reserve(8);
+ auto &Map = getContext().pImpl->MetadataUseMap;
+ for (unsigned I = 0, E = UseMap.size(); I != E; ++I)
+ Map[UseMap[I].Ref] = I;
+ } else {
+ if (LLVM_UNLIKELY(UseMap.size() == UseMap.capacity())) {
+ // Compact in place if at least half the entries are dead. Otherwise,
+ // double capacity. Because MetadataUseMap stores vector indices rather
+ // than element pointers, buffer reallocation preserves all existing
+ // indices via memcpy without touching the map.
+ if (LLVM_UNLIKELY(NumDead >= getNumUses()))
+ compact</*IsLargeMode=*/true>();
+ else
+ UseMap.reserve(UseMap.capacity() * 2);
+ }
+ }
+
+ auto &Map = getContext().pImpl->MetadataUseMap;
+ unsigned Index = UseMap.size();
+ UseMap.push_back({Ref, Owner});
+ Map[Ref] = Index;
}
void ReplaceableUses::dropRef(void *Ref) {
- bool WasErased = UseMap.erase(Ref);
- (void)WasErased;
- assert(WasErased && "Expected to drop a reference");
+ UseEntry *Entry = nullptr;
+ if (LLVM_UNLIKELY(!IsLarge)) {
+ // Search backward for temporal locality; recently added references are
+ // often dropped first.
+ for (auto It = UseMap.rbegin(), E = UseMap.rend(); It != E; ++It) {
+ if (It->Ref == Ref) {
+ Entry = &*It;
+ break;
+ }
+ }
+ assert(Entry && "Expected to find Ref");
+ } else {
+ auto &Map = getContext().pImpl->MetadataUseMap;
+ // Return early if we are tearing down the whole context.
+ if (LLVM_UNLIKELY(Map.empty()))
+ return;
+ auto It = Map.find(Ref);
+ assert(It != Map.end() && "Expected Ref in MetadataUseMap");
+ Entry = &UseMap[It->second];
+ Map.erase(It);
+ }
+
+ // Place a tombstone.
+ Entry->Ref = nullptr;
+ ++NumDead;
+
+ if (LLVM_UNLIKELY(NumDead == UseMap.size())) {
+ UseMap.clear();
+ NumDead = 0;
+ IsLarge = false;
+ }
}
void ReplaceableUses::moveRef(void *Ref, void *New, const Metadata &MD) {
- auto I = UseMap.find(Ref);
- assert(I != UseMap.end() && "Expected to move a reference");
- auto OwnerAndIndex = I->second;
- UseMap.erase(I);
- bool WasInserted = UseMap.insert(std::make_pair(New, OwnerAndIndex)).second;
- (void)WasInserted;
- assert(WasInserted && "Expected to add a reference");
+ UseEntry *Entry = nullptr;
+ if (LLVM_LIKELY(!IsLarge)) {
+ // Search backward for temporal locality; recently added references are
+ // often moved first.
+ for (auto It = UseMap.rbegin(), E = UseMap.rend(); It != E; ++It) {
+ if (It->Ref == Ref) {
+ Entry = &*It;
+ break;
+ }
+ }
+ assert(Entry && "Expected to move a reference");
+ assert(!hasRef(New) && "Cannot move to an existing reference");
+ } else {
+ auto &Map = getContext().pImpl->MetadataUseMap;
+ auto It = Map.find(Ref);
+ assert(It != Map.end() && "Expected Ref in MetadataUseMap");
+
+ unsigned Index = It->second;
+ Map.erase(It);
+ bool Inserted = Map.try_emplace(New, Index).second;
+ assert(Inserted && "Cannot move to an existing reference");
+ (void)Inserted;
+
+ Entry = &UseMap[Index];
+ }
+
+ OwnerTy Owner = Entry->Owner;
+ Entry->Ref = New;
// Check that the references are direct if there's no owner.
(void)MD;
- assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) &&
+ assert((Owner || *static_cast<Metadata **>(Ref) == &MD) &&
"Reference without owner must be direct");
- assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) &&
+ assert((Owner || *static_cast<Metadata **>(New) == &MD) &&
"Reference without owner must be direct");
}
+void ReplaceableUses::clear() {
+ if (IsLarge) {
+ auto &Map = getContext().pImpl->MetadataUseMap;
+ if (LLVM_LIKELY(!Map.empty())) {
+ for (const auto &U : UseMap) {
+ if (U.Ref)
+ Map.erase(U.Ref);
+ }
+ }
+ }
+ UseMap.clear();
+ NumDead = 0;
+ IsLarge = false;
+}
+
void ReplaceableUses::SalvageDebugInfo(const Constant &C) {
if (!C.isUsedByMetadata()) {
return;
@@ -340,14 +474,11 @@ void ReplaceableUses::SalvageDebugInfo(const Constant &C) {
auto &Store = Context.pImpl->ValuesAsMetadata;
auto I = Store.find(&C);
ValueAsMetadata *MD = I->second;
- using UseTy =
- std::pair<void *, std::pair<MetadataTracking::OwnerTy, uint64_t>>;
// Copy out uses and update value of Constant used by debug info metadata with
- // poison below
- SmallVector<UseTy, 8> Uses(MD->UseMap.begin(), MD->UseMap.end());
+ // poison below.
+ SmallVector<UseEntry, 8> Uses = MD->getLiveUses();
- for (const auto &Pair : Uses) {
- MetadataTracking::OwnerTy Owner = Pair.second.first;
+ for (const auto &[Ref, Owner] : Uses) {
if (!Owner)
continue;
// Check for MetadataAsValue.
@@ -363,7 +494,7 @@ void ReplaceableUses::SalvageDebugInfo(const Constant &C) {
continue;
if (isa<DINode>(OwnerMD)) {
OwnerMD->handleChangedOperand(
- Pair.first, ValueAsMetadata::get(PoisonValue::get(C.getType())));
+ Ref, ValueAsMetadata::get(PoisonValue::get(C.getType())));
}
}
}
@@ -373,25 +504,21 @@ void ReplaceableUses::replaceAllUsesWith(Metadata *MD) {
return;
// Copy out uses since UseMap will get touched below.
- using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
- SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
- llvm::sort(Uses, [](const UseTy &L, const UseTy &R) {
- return L.second.second < R.second.second;
- });
- for (const auto &Pair : Uses) {
+ SmallVector<UseEntry, 8> Uses = getLiveUses();
+
+ for (const auto &[Ref, Owner] : Uses) {
// Check that this Ref hasn't disappeared after RAUW (when updating a
// previous Ref).
- if (!UseMap.count(Pair.first))
+ if (!hasRef(Ref))
continue;
- OwnerTy Owner = Pair.second.first;
if (!Owner) {
// Update unowned tracking references directly.
- Metadata *&Ref = *static_cast<Metadata **>(Pair.first);
- Ref = MD;
+ Metadata *&DirectRef = *static_cast<Metadata **>(Ref);
+ dropRef(Ref);
+ DirectRef = MD;
if (MD)
- MetadataTracking::track(Ref);
- UseMap.erase(Pair.first);
+ MetadataTracking::track(DirectRef);
continue;
}
@@ -402,7 +529,7 @@ void ReplaceableUses::replaceAllUsesWith(Metadata *MD) {
}
if (auto *DVU = dyn_cast<DebugValueUser *>(Owner)) {
- DVU->handleChangedValue(Pair.first, MD);
+ DVU->handleChangedValue(Ref, MD);
continue;
}
@@ -411,7 +538,7 @@ void ReplaceableUses::replaceAllUsesWith(Metadata *MD) {
switch (OwnerMD->getMetadataID()) {
#define HANDLE_METADATA_LEAF(CLASS) \
case Metadata::CLASS##Kind: \
- cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \
+ cast<CLASS>(OwnerMD)->handleChangedOperand(Ref, MD); \
continue;
#include "llvm/IR/Metadata.def"
default:
@@ -426,19 +553,15 @@ void ReplaceableUses::resolveAllUses(bool ResolveUsers) {
return;
if (!ResolveUsers) {
- UseMap.clear();
+ clear();
return;
}
// Copy out uses since UseMap could get touched below.
- using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
- SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
- llvm::sort(Uses, [](const UseTy &L, const UseTy &R) {
- return L.second.second < R.second.second;
- });
- UseMap.clear();
- for (const auto &Pair : Uses) {
- auto Owner = Pair.second.first;
+ SmallVector<UseEntry, 8> Uses = getLiveUses();
+ clear();
+ for (const auto &U : Uses) {
+ auto Owner = U.Owner;
if (!Owner)
continue;
if (!isa<Metadata *>(Owner))
>From c4ecd265fc6912b8df8083cfd2f583d88a268b6a Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Sun, 13 Sep 2026 12:11:25 -0700
Subject: [PATCH 2/2] Fix formatting.
---
llvm/lib/IR/Metadata.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index 0c1f21a50d78c..694a62ef2eb96 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -257,8 +257,7 @@ LLVMContext &ReplaceableUses::getContext() const {
return static_cast<const ValueAsMetadata *>(this)->getContext();
}
-SmallVector<ReplaceableUses::UseEntry, 8>
-ReplaceableUses::getLiveUses() const {
+SmallVector<ReplaceableUses::UseEntry, 8> ReplaceableUses::getLiveUses() const {
SmallVector<UseEntry, 8> LiveUses;
LiveUses.reserve(getNumUses());
for (const auto &Entry : UseMap) {
More information about the llvm-commits
mailing list