[llvm] [IR] Reduce memory usage in metadata use tracking (NFC) (PR #221866)
Kazu Hirata via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 19 15:57:36 PDT 2026
https://github.com/kazutakahirata updated https://github.com/llvm/llvm-project/pull/221866
>From 185c35539390cc258b78b6d900c686b6b7b47997 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Wed, 16 Sep 2026 15:23:13 -0700
Subject: [PATCH] [IR] Reduce memory usage in metadata use tracking (NFC)
This patch reduces the memory footprint of metadata use tracking.
Currently, ReplaceableUses uses:
SmallDenseMap<void *, std::pair<OwnerTy, uint64_t>, 4>
where the uint64_t records insertion order. The problem is that
once it exceeds 2 elements, it allocates 64 buckets (~1.5 KB). In large
compilations with debug info, thousands of metadata nodes have 3 or 4
uses, leading to significant overhead.
With this patch, uses are stored directly in a SmallVector:
struct UseEntry {
void *Ref;
OwnerTy Owner;
};
SmallVector<UseEntry, 4> UseMap;
// In addRef:
UseMap.push_back({Ref, Owner});
Unlike SmallDenseMap, SmallVector stores up to 4 elements inline without
heap allocation. It also preserves insertion order, eliminating the
need to sort uses in several places.
To support this representation efficiently:
- dropRef and moveRef scan UseMap backwards from the end, taking
advantage of temporal locality since recently added references are
typically dropped or moved first.
- In replaceAllUsesWith, UseMap is reversed before iterating over the
copied use list. Because replaceAllUsesWith visits uses in original
forward order, each callback's dropRef finds its entry at the very back
of UseMap, turning linear scans into O(1) pops with zero element
shifting.
- getAllDbgVariableRecordUsers iterates over reverse(UseMap) to preserve
reverse-creation order without sorting by an explicit timestamp or ID.
Benchmark results:
Base Test
File Max RSS Max RSS Delta Wall Time
-----------------------------------------------------------------
SLPVectorizer.ii 1435 MB 1297 MB -9.60% 68.41s -> 68.02s
SemaOpenMP.ii 1461 MB 1288 MB -11.85% 55.68s -> 54.18s
X86ISelLowering.ii 974 MB 932 MB -4.30% 41.28s -> 40.97s
-----------------------------------------------------------------
Total 3870 MB 3517 MB -9.12% 165.37s -> 163.17s
Geometric Mean -8.64% -1.34%
- Compilation flags: -O3 -g -fdebug-info-for-profiling -flto=thin
- Max RSS: average of 3 runs after 1 warmup run.
- Wall Time: average of 10 runs after 1 warmup run.
Assisted-by: Antigravity
---
llvm/include/llvm/IR/Metadata.h | 9 +-
llvm/lib/IR/Metadata.cpp | 143 +++++++++++++++-----------------
2 files changed, 72 insertions(+), 80 deletions(-)
diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h
index 017fd98e704ea..b988e9306d8f4 100644
--- a/llvm/include/llvm/IR/Metadata.h
+++ b/llvm/include/llvm/IR/Metadata.h
@@ -396,9 +396,13 @@ class ReplaceableUses {
public:
using OwnerTy = MetadataTracking::OwnerTy;
+ struct UseEntry {
+ void *Ref = nullptr;
+ OwnerTy Owner = nullptr;
+ };
+
private:
- uint64_t NextIndex = 0;
- SmallDenseMap<void *, std::pair<OwnerTy, uint64_t>, 4> UseMap;
+ SmallVector<UseEntry, 4> UseMap;
protected:
~ReplaceableUses() {
@@ -429,6 +433,7 @@ class ReplaceableUses {
unsigned getNumUses() const { return UseMap.size(); }
private:
+ bool hasRef(void *Ref) const;
void addRef(void *Ref, OwnerTy Owner);
void dropRef(void *Ref);
void moveRef(void *Ref, void *New, const Metadata &MD);
diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index 28873462748b2..dd0a125064378 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -258,83 +258,76 @@ bool MetadataTracking::isReplaceable(const Metadata &MD) {
}
SmallVector<Metadata *> ReplaceableUses::getAllArgListUsers() {
- SmallVector<std::pair<OwnerTy, uint64_t> *> MDUsersWithID;
- for (auto Pair : UseMap) {
- OwnerTy Owner = Pair.second.first;
+ SmallVector<Metadata *> MDUsers;
+ for (const auto &[Ref, Owner] : UseMap) {
if (Owner.isNull())
continue;
if (!isa<Metadata *>(Owner))
continue;
Metadata *OwnerMD = cast<Metadata *>(Owner);
if (OwnerMD->getMetadataID() == Metadata::DIArgListKind)
- MDUsersWithID.push_back(&UseMap[Pair.first]);
+ MDUsers.push_back(OwnerMD);
}
- llvm::sort(MDUsersWithID, [](auto UserA, auto UserB) {
- return UserA->second < UserB->second;
- });
- SmallVector<Metadata *> MDUsers;
- for (auto *UserWithID : MDUsersWithID)
- MDUsers.push_back(cast<Metadata *>(UserWithID->first));
return MDUsers;
}
SmallVector<DbgVariableRecord *>
ReplaceableUses::getAllDbgVariableRecordUsers() {
- SmallVector<std::pair<OwnerTy, uint64_t> *> DVRUsersWithID;
- for (auto Pair : UseMap) {
- OwnerTy Owner = Pair.second.first;
- if (Owner.isNull())
- continue;
- if (!isa<DebugValueUser *>(Owner))
- continue;
- DVRUsersWithID.push_back(&UseMap[Pair.first]);
- }
// Order DbgVariableRecord users in reverse-creation order. Normal dbg.value
// users of MetadataAsValues are ordered by their UseList, i.e. reverse order
// of when they were added: we need to replicate that here. The structure of
// debug-info output depends on the ordering of intrinsics, thus we need
// to keep them consistent for comparisons sake.
- llvm::sort(DVRUsersWithID, [](auto UserA, auto UserB) {
- return UserA->second > UserB->second;
- });
SmallVector<DbgVariableRecord *> DVRUsers;
- for (auto UserWithID : DVRUsersWithID)
- DVRUsers.push_back(cast<DebugValueUser *>(UserWithID->first)->getUser());
+ for (const auto &[Ref, Owner] : reverse(UseMap)) {
+ if (Owner.isNull())
+ continue;
+ if (!isa<DebugValueUser *>(Owner))
+ continue;
+ DVRUsers.push_back(cast<DebugValueUser *>(Owner)->getUser());
+ }
return DVRUsers;
}
-void ReplaceableUses::addRef(void *Ref, OwnerTy Owner) {
- bool WasInserted =
- UseMap.insert(std::make_pair(Ref, std::make_pair(Owner, NextIndex)))
- .second;
- (void)WasInserted;
- assert(WasInserted && "Expected to add a reference");
+bool ReplaceableUses::hasRef(void *Ref) const {
+ return any_of(reverse(UseMap),
+ [Ref](const UseEntry &U) { return U.Ref == Ref; });
+}
- ++NextIndex;
- assert(NextIndex != 0 && "Unexpected overflow");
+void ReplaceableUses::addRef(void *Ref, OwnerTy Owner) {
+ assert(Ref && "Expected live reference");
+ assert(!hasRef(Ref) && "Reference already tracked");
+ UseMap.push_back({Ref, Owner});
}
void ReplaceableUses::dropRef(void *Ref) {
- bool WasErased = UseMap.erase(Ref);
- (void)WasErased;
- assert(WasErased && "Expected to drop a reference");
+ // Search backward for temporal locality; recently added references are
+ // often dropped first.
+ for (UseEntry &U : reverse(UseMap)) {
+ if (U.Ref == Ref) {
+ UseMap.erase(&U);
+ return;
+ }
+ }
+ llvm_unreachable("Expected to find Ref");
}
void ReplaceableUses::moveRef(void *Ref, void *New, const Metadata &MD) {
- auto I = UseMap.find(Ref);
- assert(I != UseMap.end() && "Expected to move a reference");
- auto OwnerAndIndex = I->second;
- UseMap.erase(I);
- bool WasInserted = UseMap.insert(std::make_pair(New, OwnerAndIndex)).second;
- (void)WasInserted;
- assert(WasInserted && "Expected to add a reference");
-
- // Check that the references are direct if there's no owner.
- (void)MD;
- assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) &&
- "Reference without owner must be direct");
- assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) &&
- "Reference without owner must be direct");
+ assert(!hasRef(New) && "Cannot move to an existing reference");
+ for (UseEntry &U : reverse(UseMap)) {
+ if (U.Ref == Ref) {
+ OwnerTy Owner = U.Owner;
+ U.Ref = New;
+ // Check that the references are direct if there's no owner.
+ (void)MD;
+ assert((Owner || *static_cast<Metadata **>(Ref) == &MD) &&
+ "Reference without owner must be direct");
+ assert((Owner || *static_cast<Metadata **>(New) == &MD) &&
+ "Reference without owner must be direct");
+ return;
+ }
+ }
+ llvm_unreachable("Expected to move a reference");
}
void ReplaceableUses::SalvageDebugInfo(const Constant &C) {
@@ -346,14 +339,11 @@ void ReplaceableUses::SalvageDebugInfo(const Constant &C) {
auto &Store = Context.pImpl->ValuesAsMetadata;
auto I = Store.find(&C);
ValueAsMetadata *MD = I->second;
- using UseTy =
- std::pair<void *, std::pair<MetadataTracking::OwnerTy, uint64_t>>;
// Copy out uses and update value of Constant used by debug info metadata with
- // poison below
- SmallVector<UseTy, 8> Uses(MD->UseMap.begin(), MD->UseMap.end());
+ // poison below.
+ SmallVector<UseEntry, 4> Uses = MD->UseMap;
- for (const auto &Pair : Uses) {
- MetadataTracking::OwnerTy Owner = Pair.second.first;
+ for (const auto &[Ref, Owner] : Uses) {
if (!Owner)
continue;
// Check for MetadataAsValue.
@@ -369,7 +359,7 @@ void ReplaceableUses::SalvageDebugInfo(const Constant &C) {
continue;
if (isa<DINode>(OwnerMD)) {
OwnerMD->handleChangedOperand(
- Pair.first, ValueAsMetadata::get(PoisonValue::get(C.getType())));
+ Ref, ValueAsMetadata::get(PoisonValue::get(C.getType())));
}
}
}
@@ -379,25 +369,26 @@ void ReplaceableUses::replaceAllUsesWith(Metadata *MD) {
return;
// Copy out uses since UseMap will get touched below.
- using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
- SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
- llvm::sort(Uses, [](const UseTy &L, const UseTy &R) {
- return L.second.second < R.second.second;
- });
- for (const auto &Pair : Uses) {
+ SmallVector<UseEntry, 4> Uses = UseMap;
+
+ // Reverse UseMap so that as we visit Uses in original forward order, each
+ // callback's dropRef finds its entry at the very back of UseMap, turning
+ // linear searches into O(1) pops.
+ std::reverse(UseMap.begin(), UseMap.end());
+
+ for (const auto &[Ref, Owner] : Uses) {
// Check that this Ref hasn't disappeared after RAUW (when updating a
// previous Ref).
- if (!UseMap.count(Pair.first))
+ if (!hasRef(Ref))
continue;
- OwnerTy Owner = Pair.second.first;
if (!Owner) {
// Update unowned tracking references directly.
- Metadata *&Ref = *static_cast<Metadata **>(Pair.first);
- Ref = MD;
+ Metadata *&DirectRef = *static_cast<Metadata **>(Ref);
+ dropRef(Ref);
+ DirectRef = MD;
if (MD)
- MetadataTracking::track(Ref);
- UseMap.erase(Pair.first);
+ MetadataTracking::track(DirectRef);
continue;
}
@@ -408,7 +399,7 @@ void ReplaceableUses::replaceAllUsesWith(Metadata *MD) {
}
if (auto *DVU = dyn_cast<DebugValueUser *>(Owner)) {
- DVU->handleChangedValue(Pair.first, MD);
+ DVU->handleChangedValue(Ref, MD);
continue;
}
@@ -417,7 +408,7 @@ void ReplaceableUses::replaceAllUsesWith(Metadata *MD) {
switch (OwnerMD->getMetadataID()) {
#define HANDLE_METADATA_LEAF(CLASS) \
case Metadata::CLASS##Kind: \
- cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \
+ cast<CLASS>(OwnerMD)->handleChangedOperand(Ref, MD); \
continue;
#include "llvm/IR/Metadata.def"
default:
@@ -436,15 +427,11 @@ void ReplaceableUses::resolveAllUses(bool ResolveUsers) {
return;
}
- // Copy out uses since UseMap could get touched below.
- using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
- SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
- llvm::sort(Uses, [](const UseTy &L, const UseTy &R) {
- return L.second.second < R.second.second;
- });
+ // Move uses out since UseMap could get touched below.
+ SmallVector<UseEntry, 4> Uses = std::move(UseMap);
UseMap.clear();
- for (const auto &Pair : Uses) {
- auto Owner = Pair.second.first;
+ for (const auto &U : Uses) {
+ auto Owner = U.Owner;
if (!Owner)
continue;
if (!isa<Metadata *>(Owner))
More information about the llvm-commits
mailing list