[llvm] [SCEV] Make PHI handling fully iterative. (PR #226624)
Eli Friedman via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 27 10:29:53 PDT 2026
https://github.com/efriedma-quic updated https://github.com/llvm/llvm-project/pull/226624
>From c66480922b1bb9011c180fe4d93af67901b57701 Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma at qti.qualcomm.com>
Date: Wed, 12 Aug 2026 14:25:21 -0700
Subject: [PATCH 1/3] [SCEV] Make PHI handling fully iterative.
Most of the patterns we support for PHI nodes can use the standard
"create the operands, then create the PHI" pattern we use for other
nodes. Loop AddRecs are unusual because we create a temporary SCEV for
the PHI so we can do some analysis on the backedge. createSCEVIter
needs some extra state to track this.
The underlying algorithm should still work essentially the same way it
worked before.
---
llvm/include/llvm/Analysis/ScalarEvolution.h | 11 +-
llvm/lib/Analysis/ScalarEvolution.cpp | 225 +++++++++++++------
2 files changed, 167 insertions(+), 69 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 33ba3a47e365ee..54a240618c79ad 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -2050,7 +2050,8 @@ class ScalarEvolution {
/// Collect operands of \p V for which SCEV expressions should be constructed
/// first. Returns a SCEV directly if it can be constructed trivially for \p
/// V.
- const SCEV *getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops);
+ const SCEV *getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops,
+ bool &AddRecPHISlowPath);
/// Returns SCEV for the first operand of a phi if all phi operands have
/// identical opcodes and operands.
@@ -2062,9 +2063,11 @@ class ScalarEvolution {
/// Helper function called from createNodeForPHI.
const SCEV *createAddRecFromPHI(PHINode *PN);
- /// A helper function for createAddRecFromPHI to handle simple cases.
- const SCEV *createSimpleAffineAddRec(PHINode *PN, Value *BEValueV,
- Value *StartValueV);
+ const SCEV *handleAddRecBackedgeForPHI(PHINode *PN, const SCEV *SymbolicName,
+ Value *BEValueV, Value *StartValueV);
+
+ /// Helper function called from createNodeForPHI.
+ const SCEV *createSimpleAffineAddRec(PHINode *PNV);
/// Helper function called from createNodeForPHI.
const SCEV *createNodeFromSelectLikePHI(PHINode *PN);
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index a57c489b534f1a..a7bf009a0ddbdb 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -5696,18 +5696,82 @@ getNoWrapFlagsForGEP(GEPOperator *GEP, const SCEV *Accum, ScalarEvolution &SE) {
return Flags;
}
+/// technique for finding the AddRec expression.
+Value *canCreateSimpleAffineAddRec(PHINode *PN, Value *BEValueV,
+ Value *StartValueV, const DataLayout &DL,
+ AssumptionCache &AC, DominatorTree &DT,
+ LoopInfo &LI) {
+ const Loop *L = LI.getLoopFor(PN->getParent());
+ assert(L && L->getHeader() == PN->getParent());
+ assert(BEValueV && StartValueV);
+
+ if (auto BO = MatchBinaryOp(BEValueV, DL, AC, DT, PN)) {
+ if (BO->Opcode != Instruction::Add)
+ return nullptr;
+
+ if (BO->LHS == PN && L->isLoopInvariant(BO->RHS))
+ return BO->RHS;
+ else if (BO->RHS == PN && L->isLoopInvariant(BO->LHS))
+ return BO->LHS;
+
+ return nullptr;
+ }
+
+ // Handle pointer induction variable: PN = PHI(Start, gep PN,
+ // LoopInvariant).
+ auto *GEP = dyn_cast<GEPOperator>(BEValueV);
+ if (!GEP || GEP->getPointerOperand() != PN || GEP->getNumIndices() != 1)
+ return nullptr;
+ Value *Idx = *GEP->idx_begin();
+ if (!L->isLoopInvariant(Idx))
+ return nullptr;
+
+ return Idx;
+}
+
+// Compute the initial and backedge values for a PHI node in a loop header.
+static std::pair<Value *, Value *> valuesForAddRecFromPHI(LoopInfo &LI,
+ PHINode *PN) {
+ const Loop *L = LI.getLoopFor(PN->getParent());
+ if (!L || L->getHeader() != PN->getParent())
+ return {};
+
+ // The loop may have multiple entrances or multiple exits; we can analyze
+ // this phi as an addrec if it has a unique entry value and a unique
+ // backedge value.
+ Value *BEValueV = nullptr, *StartValueV = nullptr;
+ for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
+ Value *V = PN->getIncomingValue(i);
+ if (L->contains(PN->getIncomingBlock(i))) {
+ if (!BEValueV) {
+ BEValueV = V;
+ } else if (BEValueV != V) {
+ BEValueV = nullptr;
+ break;
+ }
+ } else if (!StartValueV) {
+ StartValueV = V;
+ } else if (StartValueV != V) {
+ StartValueV = nullptr;
+ break;
+ }
+ }
+ return {BEValueV, StartValueV};
+}
+
/// A helper function for createAddRecFromPHI to handle simple cases.
///
/// This function tries to find an AddRec expression for the simplest (yet most
/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
/// If it fails, createAddRecFromPHI will use a more general, but slow,
/// technique for finding the AddRec expression.
-const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
- Value *BEValueV,
- Value *StartValueV) {
+const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN) {
+ auto [BEValueV, StartValueV] = valuesForAddRecFromPHI(LI, PN);
+ if (!BEValueV || !StartValueV)
+ return nullptr;
+
const Loop *L = LI.getLoopFor(PN->getParent());
assert(L && L->getHeader() == PN->getParent());
- assert(BEValueV && StartValueV);
const SCEV *Accum = nullptr;
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
@@ -5763,49 +5827,14 @@ const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
return PHISCEV;
}
-const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
- const Loop *L = LI.getLoopFor(PN->getParent());
- if (!L || L->getHeader() != PN->getParent())
- return nullptr;
-
- // The loop may have multiple entrances or multiple exits; we can analyze
- // this phi as an addrec if it has a unique entry value and a unique
- // backedge value.
- Value *BEValueV = nullptr, *StartValueV = nullptr;
- for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
- Value *V = PN->getIncomingValue(i);
- if (L->contains(PN->getIncomingBlock(i))) {
- if (!BEValueV) {
- BEValueV = V;
- } else if (BEValueV != V) {
- BEValueV = nullptr;
- break;
- }
- } else if (!StartValueV) {
- StartValueV = V;
- } else if (StartValueV != V) {
- StartValueV = nullptr;
- break;
- }
- }
- if (!BEValueV || !StartValueV)
- return nullptr;
-
- assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
- "PHI node already processed?");
-
- // First, try to find AddRec expression without creating a fictituos symbolic
- // value for PN.
- if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
- return S;
-
- // Handle PHI node value symbolically.
- const SCEV *SymbolicName = getUnknown(PN);
- insertValueToMap(PN, SymbolicName);
-
+const SCEV *ScalarEvolution::handleAddRecBackedgeForPHI(
+ PHINode *PN, const SCEV *SymbolicName, Value *BEValueV,
+ Value *StartValueV) {
// Using this symbolic name for the PHI, analyze the value coming around
// the back-edge.
- const SCEV *BEValue = getSCEV(BEValueV);
+ const Loop *L = LI.getLoopFor(PN->getParent());
+ const SCEV *BEValue = getExistingSCEV(BEValueV);
+ assert(BEValue && "createSCEVIter should have created this");
// NOTE: If BEValue is loop invariant, we know that the PHI node just
// has a special value for the first iteration of the loop.
@@ -6021,7 +6050,7 @@ ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
}
const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
- if (const SCEV *S = createAddRecFromPHI(PN))
+ if (const SCEV *S = createSimpleAffineAddRec(PN))
return S;
// We do not allow simplifying phi (undef, X) to X here, to avoid reusing the
@@ -6037,8 +6066,9 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
if (const SCEV *S = createNodeFromSelectLikePHI(PN))
return S;
- // If it's not a loop phi, we can't handle it yet.
- return getUnknown(PN);
+ // We don't know how to handle this here. Note that createSCEVIter has
+ // a special case for backedge PHIs.
+ return nullptr;
}
bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind,
@@ -7536,47 +7566,99 @@ bool ScalarEvolution::loopIsFiniteByAssumption(const Loop *L) {
const SCEV *ScalarEvolution::createSCEVIter(Value *V) {
// Worklist item with a Value and a bool indicating whether all operands have
// been visited already.
- using PointerTy = PointerIntPair<Value *, 1, bool>;
+ enum class StackState {
+ Unvisited,
+ Visited,
+ PHIBackedge,
+ PHIInitialValue,
+ };
+ using PointerTy = PointerIntPair<Value *, 2, StackState>;
SmallVector<PointerTy> Stack;
- Stack.emplace_back(V, false);
+ Stack.emplace_back(V, StackState::Unvisited);
while (!Stack.empty()) {
auto E = Stack.back();
Value *CurV = E.getPointer();
+ if (E.getInt() == StackState::PHIBackedge) {
+ // Try to construct the backedge; either we succeed, or fallback to
+ // SCEVUnknown.
+ PHINode *PN = cast<PHINode>(CurV);
+ auto [BEValueV, StartValueV] = valuesForAddRecFromPHI(LI, PN);
+ const SCEV *CreatedSCEV;
+ if (const SCEV *S = handleAddRecBackedgeForPHI(PN, getExistingSCEV(CurV),
+ BEValueV, StartValueV)) {
+ CreatedSCEV = S;
+ } else {
+ CreatedSCEV = getUnknown(PN);
+ }
+ insertValueToMap(PN, CreatedSCEV);
+ Stack.pop_back();
+ continue;
+ }
+
if (getExistingSCEV(CurV)) {
Stack.pop_back();
continue;
}
+ if (E.getInt() == StackState::PHIInitialValue) {
+ // We have a PHI where we handled the initial value. First query the fast
+ // path in case this actually matches a different pattern.
+ PHINode *PN = cast<PHINode>(CurV);
+ if (const SCEV *S = createNodeForPHI(PN)) {
+ insertValueToMap(CurV, S);
+ Stack.pop_back();
+ continue;
+ }
+ // Try the slow path of creating a SCEVUnknown for the PHI, and
+ // computing a SCEV for the backedge.
+ //
+ // FIXME: This is sensitive to the query order in some edge cases.
+ // The node we create here can block analysis of a PHI node in a parent
+ // loop's header.
+ const SCEV *SymbolicName = getUnknown(PN);
+ insertValueToMap(PN, SymbolicName);
+ Stack.back().setInt(StackState::PHIBackedge);
+ auto [BEValueV, StartValueV] = valuesForAddRecFromPHI(LI, PN);
+ Stack.emplace_back(BEValueV, StackState::Unvisited);
+ continue;
+ }
+
SmallVector<Value *> Ops;
const SCEV *CreatedSCEV = nullptr;
- // If all operands have been visited already, create the SCEV.
- if (E.getInt()) {
+ bool AddRecPHISlowPath = false;
+
+ if (E.getInt() == StackState::Visited) {
+ // If all operands have been visited already, create the SCEV.
CreatedSCEV = createSCEV(CurV);
} else {
// Otherwise get the operands we need to create SCEV's for before creating
// the SCEV for CurV. If the SCEV for CurV can be constructed trivially,
// just use it.
- CreatedSCEV = getOperandsToCreate(CurV, Ops);
+ CreatedSCEV = getOperandsToCreate(CurV, Ops, AddRecPHISlowPath);
}
if (CreatedSCEV) {
insertValueToMap(CurV, CreatedSCEV);
Stack.pop_back();
} else {
- Stack.back().setInt(true);
+ if (AddRecPHISlowPath)
+ Stack.back().setInt(StackState::PHIInitialValue);
+ else
+ Stack.back().setInt(StackState::Visited);
// Queue its operands which need to be constructed.
for (Value *Op : Ops)
- Stack.emplace_back(Op, false);
+ Stack.emplace_back(Op, StackState::Unvisited);
}
}
return getExistingSCEV(V);
}
-const SCEV *
-ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
+const SCEV *ScalarEvolution::getOperandsToCreate(Value *V,
+ SmallVectorImpl<Value *> &Ops,
+ bool &AddRecPHISlowPath) {
if (!isSCEVable(V->getType()))
return getUnknown(V);
@@ -7714,7 +7796,6 @@ ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
if (BinaryOperator *BO = getCommonInstForPHI(cast<PHINode>(U))) {
assert(BO);
Ops.push_back(BO);
- return nullptr;
}
// The third is createNodeFromSelectLikePHI; this takes a PHI which
// is equivalent to a select, and analyzes it like a select.
@@ -7731,18 +7812,30 @@ ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
Ops.push_back(Cond);
Ops.push_back(LHS);
Ops.push_back(RHS);
- return nullptr;
}
}
- // The fourth way is createAddRecFromPHI. It's complicated to handle here,
- // so just construct it recursively.
- //
+ // The fourth way is an AddRec for a loop. We have a fast path using
+ // canCreateSimpleAffineAddRec. AddRecPHISlowPath requests special
+ // handling in createSCEVIter.
+ {
+ auto [BEValueV, StartValueV] =
+ valuesForAddRecFromPHI(LI, cast<PHINode>(U));
+ if (BEValueV && StartValueV) {
+ if (Value *Increment = canCreateSimpleAffineAddRec(
+ cast<PHINode>(U), BEValueV, StartValueV, DL, AC, DT, LI)) {
+ Ops.push_back(StartValueV);
+ Ops.push_back(Increment);
+ } else {
+ Ops.push_back(StartValueV);
+ AddRecPHISlowPath = true;
+ }
+ }
+ }
// In addition to getNodeForPHI, also construct nodes which might be needed
- // by getRangeRef.
+ // by getRangeRef() on a SCEVUnknown for a PHI.
if (RangeRefPHIAllowedOperands(DT, cast<PHINode>(U))) {
for (Value *V : cast<PHINode>(U)->operands())
Ops.push_back(V);
- return nullptr;
}
return nullptr;
@@ -8253,7 +8346,9 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
return createNodeForGEP(cast<GEPOperator>(U));
case Instruction::PHI:
- return createNodeForPHI(cast<PHINode>(U));
+ if (const SCEV *S = createNodeForPHI(cast<PHINode>(U)))
+ return S;
+ return getUnknown(V);
case Instruction::Select:
return createNodeForSelectOrPHI(U, U->getOperand(0), U->getOperand(1),
>From 101aefaa9d5070ffc92a3ca52421317c1ffa9346 Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma at qti.qualcomm.com>
Date: Sun, 27 Sep 2026 10:21:51 -0700
Subject: [PATCH 2/3] Update comments.
---
llvm/include/llvm/Analysis/ScalarEvolution.h | 18 ++++++++----------
llvm/lib/Analysis/ScalarEvolution.cpp | 6 +++---
2 files changed, 11 insertions(+), 13 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 54a240618c79ad..64c70fa8ee46a0 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -2060,9 +2060,6 @@ class ScalarEvolution {
/// Provide the special handling we need to analyze PHI SCEVs.
const SCEV *createNodeForPHI(PHINode *PN);
- /// Helper function called from createNodeForPHI.
- const SCEV *createAddRecFromPHI(PHINode *PN);
-
const SCEV *handleAddRecBackedgeForPHI(PHINode *PN, const SCEV *SymbolicName,
Value *BEValueV, Value *StartValueV);
@@ -2505,13 +2502,14 @@ class ScalarEvolution {
/// add recurrence on the loop \p L.
bool isAddRecNeverPoison(const Instruction *I, const Loop *L);
- /// Similar to createAddRecFromPHI, but with the additional flexibility of
- /// suggesting runtime overflow checks in case casts are encountered.
- /// If successful, the analysis records that for this loop, \p SymbolicPHI,
- /// which is the UnknownSCEV currently representing the PHI, can be rewritten
- /// into an AddRec, assuming some predicates; The function then returns the
- /// AddRec and the predicates as a pair, and caches this pair in
- /// PredicatedSCEVRewrites.
+ /// Similar to handleAddRecBackedgeForPHI, but with the additional
+ /// flexibility of suggesting runtime overflow checks in case casts are
+ /// encountered. If successful, the analysis records that for this loop,
+ /// \p SymbolicPHI, which is the UnknownSCEV currently representing the PHI,
+ /// can be rewritten into an AddRec, assuming some predicates; The function
+ /// then returns the AddRec and the predicates as a pair, and caches this
+ /// pair in PredicatedSCEVRewrites.
+ ///
/// If the analysis is not successful, a mapping from the \p SymbolicPHI to
/// itself (with no predicates) is recorded, and a nullptr with an empty
/// predicates vector is returned as a pair.
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index a7bf009a0ddbdb..a3b95de0e02e23 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -5403,7 +5403,7 @@ static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
// (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
// which correspond to a phi->trunc->add->sext/zext->phi update chain.
//
-// 3) Outline common code with createAddRecFromPHI to avoid duplication.
+// 3) Outline common code with handleAddRecBackedgeForPHI to avoid duplication.
std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
SmallVector<const SCEVPredicate *, 3> Predicates;
@@ -5759,11 +5759,11 @@ static std::pair<Value *, Value *> valuesForAddRecFromPHI(LoopInfo &LI,
return {BEValueV, StartValueV};
}
-/// A helper function for createAddRecFromPHI to handle simple cases.
+/// A helper function to construct simple AddRecs.
///
/// This function tries to find an AddRec expression for the simplest (yet most
/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
-/// If it fails, createAddRecFromPHI will use a more general, but slow,
+/// If it fails, handleAddRecBackedgeForPHI will use a more general, but slow,
/// technique for finding the AddRec expression.
const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN) {
auto [BEValueV, StartValueV] = valuesForAddRecFromPHI(LI, PN);
>From b851dbbcfd5d60b7c2edb3edd80ac4535e1f4427 Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma at qti.qualcomm.com>
Date: Sun, 27 Sep 2026 10:29:28 -0700
Subject: [PATCH 3/3] Fix clang-format
---
llvm/lib/Analysis/ScalarEvolution.cpp | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index a3b95de0e02e23..e58c40412fe72a 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -5361,26 +5361,26 @@ static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
//
// Example usage scenario:
// Say the Rewriter is called for the following SCEV:
-// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
+// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
// where:
-// %X = phi i64 (%Start, %BEValue)
+// %X = phi i64 (%Start, %BEValue)
// It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
// and call this function with %SymbolicPHI = %X.
//
// The analysis will find that the value coming around the backedge has
// the following SCEV:
-// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
+// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
// Upon concluding that this matches the desired pattern, the function
// will return the pair {NewAddRec, SmallPredsVec} where:
-// NewAddRec = {%Start,+,%Step}
-// SmallPredsVec = {P1, P2, P3} as follows:
-// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
-// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
-// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
+// NewAddRec = {%Start,+,%Step}
+// SmallPredsVec = {P1, P2, P3} as follows:
+// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
+// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
+// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
// The returned pair means that SymbolicPHI can be rewritten into NewAddRec
// under the predicates {P1,P2,P3}.
// This predicated rewrite will be cached in PredicatedSCEVRewrites:
-// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
+// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
//
// TODO's:
//
More information about the llvm-commits
mailing list