[llvm] [SCEV] Look up uniqued nodes by using interned profile. (NFC) (PR #217571)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 03:09:42 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-analysis
@llvm/pr-subscribers-llvm-support
Author: Florian Hahn (fhahn)
<details>
<summary>Changes</summary>
Each SCEV node already stores an interned profile/hash. Add an overload for FindNodeOrInsertPos which takes a match callback; SCEV uses this to match against the interned ID, which saves unnecessary pointer walks to hash/compare.
Improves compile-time in most cases. The odd thing out is that there are almost no gains in the stage2 build, presumably because the system compiler produces better results (faster stage1 clang), but stage2 builds with latest clang and there's much less benefit there. This may be a missed optimization in stage2 clang (or better optimizations in latest clang)
* stage1-O3: -0.11%
* stage1-ReleaseThinLTO: -0.11%
* stage1-ReleaseLTO-g: -0.11%
* stage1-aarch64-O3: -0.12%
* stage2-O3: -0.01%
* clang-build: -0.07%
https://llvm-compile-time-tracker.com/compare.php?from=7fd523496fc9e05ed5b0eabbfaac951d7e7f5dc7&to=1c8a4a886af918d20c7e4aaa0676d84f27174af8&stat=instructions:u
---
Full diff: https://github.com/llvm/llvm-project/pull/217571.diff
4 Files Affected:
- (modified) llvm/include/llvm/ADT/FoldingSet.h (+46)
- (modified) llvm/include/llvm/Analysis/ScalarEvolution.h (+6)
- (modified) llvm/lib/Analysis/ScalarEvolution.cpp (+35-22)
- (modified) llvm/lib/Support/FoldingSet.cpp (+6-30)
``````````diff
diff --git a/llvm/include/llvm/ADT/FoldingSet.h b/llvm/include/llvm/ADT/FoldingSet.h
index 1b0e967c8308d..765bc441b36a1 100644
--- a/llvm/include/llvm/ADT/FoldingSet.h
+++ b/llvm/include/llvm/ADT/FoldingSet.h
@@ -296,6 +296,8 @@ class FoldingSetNodeID {
/// facilitate node removal.
///
class FoldingSetBase {
+ friend class FoldingSetIteratorImpl;
+
protected:
/// Array of bucket chains.
void **Buckets;
@@ -374,6 +376,25 @@ class FoldingSetBase {
void GrowBucketCount(unsigned NewBucketCount, const FoldingSetInfo &Info);
protected:
+ /// Return the hash bucket for \p Hash.
+ void **getBucketFor(unsigned Hash) const {
+ // NumBuckets is always a power of 2.
+ return Buckets + (Hash & (NumBuckets - 1));
+ }
+
+ /// In order to save space, each bucket is a singly-linked-list. In order to
+ /// make deletion more efficient, we make the list circular, so we can delete
+ /// a node without computing its hash. The problem with this is that the start
+ /// of the hash buckets are not Nodes. If \p NextInBucketPtr is a bucket
+ /// pointer, this method returns null.
+ static Node *GetNextPtr(void *NextInBucketPtr) {
+ // The low bit is set if this is the pointer back to the bucket.
+ if (reinterpret_cast<intptr_t>(NextInBucketPtr) & 1)
+ return nullptr;
+
+ return static_cast<Node *>(NextInBucketPtr);
+ }
+
// The below methods are protected to encourage subclasses to provide a more
// type-safe API.
@@ -505,6 +526,31 @@ class FoldingSetImpl : public FoldingSetBase, public Trait::ContextStorage {
const_iterator begin() const { return const_iterator(Buckets); }
const_iterator end() const { return const_iterator(Buckets + NumBuckets); }
+ /// Look up the node matching \p ID, using \p IsMatch to compare a candidate
+ /// node against it. If there is no such node, return null and set
+ /// \p InsertPos to the insertion token to pass to InsertNode(); the token is
+ /// only valid until the set is modified.
+ ///
+ /// This is an alternative to the FoldingSetInfo-based overload for clients
+ /// whose nodes can be compared against an ID directly.
+ template <typename FnT>
+ T *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos,
+ FnT IsMatch) {
+ void **Bucket = getBucketFor(ID.ComputeHash());
+ for (void *Probe = *Bucket; Node *N = GetNextPtr(Probe);
+ Probe = N->getNextInBucket()) {
+ T *TN = static_cast<T *>(N);
+ if (IsMatch(*TN)) {
+ InsertPos = nullptr;
+ return TN;
+ }
+ }
+
+ // Didn't find the node, return null with the bucket as the InsertPos.
+ InsertPos = Bucket;
+ return nullptr;
+ }
+
/// Increase the number of buckets such that adding the \p EltCount th node
/// won't cause a rebucket operation. reserve is permitted to allocate more
/// space than requested by EltCount.
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index d17c21ea3401e..b4ee92746af46 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -326,6 +326,9 @@ class SCEV : public FoldingSetNode {
/// stream. This should really only be used for debugging purposes.
LLVM_ABI void print(raw_ostream &OS) const;
+ /// Return true if \p ID is this node's interned uniquing profile.
+ bool hasProfile(const FoldingSetNodeID &ID) const { return ID == FastID; }
+
/// This method is used for debugging.
LLVM_ABI void dump() const;
@@ -399,6 +402,9 @@ class SCEVPredicate : public FoldingSetNode {
SCEVPredicateKind getKind() const { return Kind; }
+ /// Return true if \p ID is this node's interned uniquing profile.
+ bool hasProfile(const FoldingSetNodeID &ID) const { return ID == FastID; }
+
/// Returns the estimated complexity of this predicate. This is roughly
/// measured in the number of run-time checks required.
virtual unsigned getComplexity() const { return 1; }
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 35d650377d719..00ae00fd44af9 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -506,6 +506,16 @@ bool SCEVCouldNotCompute::classof(const SCEV *S) {
return S->getSCEVType() == scCouldNotCompute;
}
+/// Look up the node with profile \p ID in \p Set. On a miss, set \p InsertPos
+/// for a subsequent Set.InsertNode() and return nullptr.
+template <typename NodeTy>
+static NodeTy *findUniqued(FoldingSet<NodeTy> &Set, const FoldingSetNodeID &ID,
+ void *&InsertPos) {
+ // The nodes intern their profile, so compare against it directly.
+ return Set.FindNodeOrInsertPos(
+ ID, InsertPos, [&ID](const NodeTy &N) { return N.hasProfile(ID); });
+}
+
const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
auto &Entry = ConstantSCEVs[V];
if (Entry)
@@ -516,7 +526,7 @@ const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
ID.AddPointer(V);
void *IP = nullptr;
if (SCEVConstant *S =
- static_cast<SCEVConstant *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)))
+ static_cast<SCEVConstant *>(findUniqued(UniqueSCEVs, ID, IP)))
return Entry = S;
SCEVConstant *S =
new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V);
@@ -543,7 +553,7 @@ const SCEV *ScalarEvolution::getVScale(Type *Ty) {
ID.AddInteger(scVScale);
ID.AddPointer(Ty);
void *IP = nullptr;
- if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
+ if (const SCEV *S = findUniqued(UniqueSCEVs, ID, IP))
return S;
SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(SCEVAllocator), Ty);
UniqueSCEVs.InsertNode(S, IP);
@@ -1142,7 +1152,7 @@ const SCEV *ScalarEvolution::getPtrToAddrExpr(const SCEV *Op) {
ID.AddPointer(U);
ID.AddPointer(Ty);
void *IP = nullptr;
- if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
+ if (const SCEV *S = findUniqued(UniqueSCEVs, ID, IP))
return S;
SCEV *S = new (SCEVAllocator)
SCEVPtrToAddrExpr(ID.Intern(SCEVAllocator), U, Ty);
@@ -1171,7 +1181,8 @@ const SCEV *ScalarEvolution::getTruncateExpr(SCEVUse Op, Type *Ty,
ID.AddPointer(Op.getOpaqueValue());
ID.AddPointer(Ty);
void *IP = nullptr;
- if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
+ if (const SCEV *S = findUniqued(UniqueSCEVs, ID, IP))
+ return S;
// Fold if the operand is constant.
if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
@@ -1225,7 +1236,7 @@ const SCEV *ScalarEvolution::getTruncateExpr(SCEVUse Op, Type *Ty,
// Although we checked in the beginning that ID is not in the cache, it is
// possible that during recursion and different modification ID was inserted
// into the cache. So if we find it, just return it.
- if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
+ if (const SCEV *S = findUniqued(UniqueSCEVs, ID, IP))
return S;
}
@@ -1497,7 +1508,7 @@ bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
ID.AddPointer(L);
void *IP = nullptr;
const auto *PreAR =
- static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
+ static_cast<SCEVAddRecExpr *>(findUniqued(UniqueSCEVs, ID, IP));
// Give up if we don't already have the add recurrence we need because
// actually constructing an add recurrence is relatively expensive.
@@ -1628,7 +1639,8 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(SCEVUse Op, Type *Ty,
ID.AddPointer(Op.getOpaqueValue());
ID.AddPointer(Ty);
void *IP = nullptr;
- if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
+ if (const SCEV *S = findUniqued(UniqueSCEVs, ID, IP))
+ return S;
if (Depth > MaxCastDepth) {
SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
Op, Ty);
@@ -1914,7 +1926,8 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(SCEVUse Op, Type *Ty,
// The cast wasn't folded; create an explicit cast node.
// Recompute the insert position, as it may have been invalidated.
- if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
+ if (const SCEV *S = findUniqued(UniqueSCEVs, ID, IP))
+ return S;
SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
@@ -1983,7 +1996,8 @@ const SCEV *ScalarEvolution::getSignExtendExprImpl(SCEVUse Op, Type *Ty,
ID.AddPointer(Op.getOpaqueValue());
ID.AddPointer(Ty);
void *IP = nullptr;
- if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
+ if (const SCEV *S = findUniqued(UniqueSCEVs, ID, IP))
+ return S;
// Limit recursion depth.
if (Depth > MaxCastDepth) {
SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
@@ -2177,7 +2191,8 @@ const SCEV *ScalarEvolution::getSignExtendExprImpl(SCEVUse Op, Type *Ty,
// The cast wasn't folded; create an explicit cast node.
// Recompute the insert position, as it may have been invalidated.
- if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
+ if (const SCEV *S = findUniqued(UniqueSCEVs, ID, IP))
+ return S;
SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
@@ -3032,8 +3047,7 @@ const SCEV *ScalarEvolution::getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
for (SCEVUse Op : Ops)
ID.AddPointer(Op.getOpaqueValue());
void *IP = nullptr;
- SCEVAddExpr *S =
- static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
+ SCEVAddExpr *S = static_cast<SCEVAddExpr *>(findUniqued(UniqueSCEVs, ID, IP));
if (!S) {
SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
llvm::uninitialized_copy(Ops, O);
@@ -3057,7 +3071,7 @@ const SCEV *ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<SCEVUse> Ops,
ID.AddPointer(L);
void *IP = nullptr;
SCEVAddRecExpr *S =
- static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
+ static_cast<SCEVAddRecExpr *>(findUniqued(UniqueSCEVs, ID, IP));
if (!S) {
SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
llvm::uninitialized_copy(Ops, O);
@@ -3079,8 +3093,7 @@ const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
for (SCEVUse Op : Ops)
ID.AddPointer(Op.getOpaqueValue());
void *IP = nullptr;
- SCEVMulExpr *S =
- static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
+ SCEVMulExpr *S = static_cast<SCEVMulExpr *>(findUniqued(UniqueSCEVs, ID, IP));
if (!S) {
SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
llvm::uninitialized_copy(Ops, O);
@@ -3100,7 +3113,7 @@ const SCEV *ScalarEvolution::getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS) {
ID.AddPointer(LHS.getOpaqueValue());
ID.AddPointer(RHS.getOpaqueValue());
void *IP = nullptr;
- SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
+ SCEV *S = findUniqued(UniqueSCEVs, ID, IP);
if (!S) {
S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator), LHS, RHS);
UniqueSCEVs.InsertNode(S, IP);
@@ -3890,7 +3903,7 @@ SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
for (SCEVUse Op : Ops)
ID.AddPointer(Op.getOpaqueValue());
void *IP = nullptr;
- return UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
+ return findUniqued(UniqueSCEVs, ID, IP);
}
const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
@@ -4012,7 +4025,7 @@ const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
for (SCEVUse Op : Ops)
ID.AddPointer(Op.getOpaqueValue());
void *IP = nullptr;
- const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
+ const SCEV *ExistingSCEV = findUniqued(UniqueSCEVs, ID, IP);
if (ExistingSCEV)
return ExistingSCEV;
SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
@@ -4399,7 +4412,7 @@ ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
for (SCEVUse Op : Ops)
ID.AddPointer(Op.getOpaqueValue());
void *IP = nullptr;
- const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
+ const SCEV *ExistingSCEV = findUniqued(UniqueSCEVs, ID, IP);
if (ExistingSCEV)
return ExistingSCEV;
@@ -4491,7 +4504,7 @@ const SCEV *ScalarEvolution::getUnknown(Value *V) {
ID.AddInteger(scUnknown);
ID.AddPointer(V);
void *IP = nullptr;
- if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) {
+ if (SCEV *S = findUniqued(UniqueSCEVs, ID, IP)) {
assert(cast<SCEVUnknown>(S)->getValue() == V &&
"Stale SCEVUnknown in uniquing map!");
return S;
@@ -15157,7 +15170,7 @@ ScalarEvolution::getComparePredicate(const ICmpInst::Predicate Pred,
ID.AddPointer(LHS);
ID.AddPointer(RHS);
void *IP = nullptr;
- if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
+ if (const auto *S = findUniqued(UniquePreds, ID, IP))
return S;
SCEVComparePredicate *Eq = new (SCEVAllocator)
SCEVComparePredicate(ID.Intern(SCEVAllocator), Pred, LHS, RHS);
@@ -15174,7 +15187,7 @@ const SCEVPredicate *ScalarEvolution::getWrapPredicate(
ID.AddPointer(AR);
ID.AddInteger(AddedFlags);
void *IP = nullptr;
- if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
+ if (const auto *S = findUniqued(UniquePreds, ID, IP))
return S;
auto *OF = new (SCEVAllocator)
SCEVWrapPredicate(ID.Intern(SCEVAllocator), AR, AddedFlags);
diff --git a/llvm/lib/Support/FoldingSet.cpp b/llvm/lib/Support/FoldingSet.cpp
index 1423bc51b78b6..27254d632a451 100644
--- a/llvm/lib/Support/FoldingSet.cpp
+++ b/llvm/lib/Support/FoldingSet.cpp
@@ -133,20 +133,6 @@ FoldingSetNodeID::Intern(BumpPtrAllocator &Allocator) const {
//===----------------------------------------------------------------------===//
/// Helper functions for FoldingSetBase.
-/// GetNextPtr - In order to save space, each bucket is a
-/// singly-linked-list. In order to make deletion more efficient, we make
-/// the list circular, so we can delete a node without computing its hash.
-/// The problem with this is that the start of the hash buckets are not
-/// Nodes. If NextInBucketPtr is a bucket pointer, this method returns null:
-/// use GetBucketPtr when this happens.
-static FoldingSetBase::Node *GetNextPtr(void *NextInBucketPtr) {
- // The low bit is set if this is the pointer back to the bucket.
- if (reinterpret_cast<intptr_t>(NextInBucketPtr) & 1)
- return nullptr;
-
- return static_cast<FoldingSetBase::Node *>(NextInBucketPtr);
-}
-
/// GetBucketPtr - Provides a casting of a bucket pointer for isNode
/// testing.
static void **GetBucketPtr(void *NextInBucketPtr) {
@@ -155,14 +141,6 @@ static void **GetBucketPtr(void *NextInBucketPtr) {
return reinterpret_cast<void **>(Ptr & ~intptr_t(1));
}
-/// GetBucketFor - Hash the specified node ID and return the hash bucket for
-/// the specified ID.
-static void **GetBucketFor(unsigned Hash, void **Buckets, unsigned NumBuckets) {
- // NumBuckets is always a power of 2.
- unsigned BucketNum = Hash & (NumBuckets - 1);
- return Buckets + BucketNum;
-}
-
/// AllocateBuckets - Allocate initialized bucket memory.
static void **AllocateBuckets(unsigned NumBuckets) {
void **Buckets =
@@ -234,8 +212,7 @@ void FoldingSetBase::GrowBucketCount(unsigned NewBucketCount,
// Insert the node into the new bucket, after recomputing the hash.
Tmp.InsertNode(
NodeInBucket,
- GetBucketFor(Info.ComputeNodeHash(this, NodeInBucket, TempID),
- Tmp.Buckets, Tmp.NumBuckets),
+ Tmp.getBucketFor(Info.ComputeNodeHash(this, NodeInBucket, TempID)),
Info);
TempID.clear();
}
@@ -256,7 +233,7 @@ void FoldingSetBase::reserve(unsigned EltCount, const FoldingSetInfo &Info) {
FoldingSetBase::Node *FoldingSetBase::FindNodeOrInsertPos(
const FoldingSetNodeID &ID, void *&InsertPos, const FoldingSetInfo &Info) {
unsigned IDHash = ID.ComputeHash();
- void **Bucket = GetBucketFor(IDHash, Buckets, NumBuckets);
+ void **Bucket = getBucketFor(IDHash);
void *Probe = *Bucket;
InsertPos = nullptr;
@@ -282,8 +259,7 @@ void FoldingSetBase::InsertNode(Node *N, void *InsertPos,
if (NumNodes + 1 > capacity()) {
GrowBucketCount(NumBuckets * 2, Info);
FoldingSetNodeID TempID;
- InsertPos = GetBucketFor(Info.ComputeNodeHash(this, N, TempID), Buckets,
- NumBuckets);
+ InsertPos = getBucketFor(Info.ComputeNodeHash(this, N, TempID));
}
++NumNodes;
@@ -360,7 +336,7 @@ FoldingSetBase::GetOrInsertNode(Node *N, const FoldingSetInfo &Info) {
FoldingSetIteratorImpl::FoldingSetIteratorImpl(void **Bucket) {
// Skip to the first non-null non-self-cycle bucket.
while (*Bucket != reinterpret_cast<void *>(-1) &&
- (!*Bucket || !GetNextPtr(*Bucket)))
+ (!*Bucket || !FoldingSetBase::GetNextPtr(*Bucket)))
++Bucket;
NodePtr = static_cast<FoldingSetNode *>(*Bucket);
@@ -370,7 +346,7 @@ void FoldingSetIteratorImpl::advance() {
// If there is another link within this bucket, go to it.
void *Probe = NodePtr->getNextInBucket();
- if (FoldingSetNode *NextNodeInBucket = GetNextPtr(Probe))
+ if (FoldingSetNode *NextNodeInBucket = FoldingSetBase::GetNextPtr(Probe))
NodePtr = NextNodeInBucket;
else {
// Otherwise, this is the last link in this bucket.
@@ -380,7 +356,7 @@ void FoldingSetIteratorImpl::advance() {
do {
++Bucket;
} while (*Bucket != reinterpret_cast<void *>(-1) &&
- (!*Bucket || !GetNextPtr(*Bucket)));
+ (!*Bucket || !FoldingSetBase::GetNextPtr(*Bucket)));
NodePtr = static_cast<FoldingSetNode *>(*Bucket);
}
``````````
</details>
https://github.com/llvm/llvm-project/pull/217571
More information about the llvm-commits
mailing list