[llvm] 08169f5 - [SCEV] Return a SCEVUse from getMulExpr and propagate use flags. (#221486)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 15 01:05:41 PDT 2026
Author: Florian Hahn
Date: 2026-09-15T09:05:35+01:00
New Revision: 08169f5fb1b7386002cdb66e52192580fc0fcf24
URL: https://github.com/llvm/llvm-project/commit/08169f5fb1b7386002cdb66e52192580fc0fcf24
DIFF: https://github.com/llvm/llvm-project/commit/08169f5fb1b7386002cdb66e52192580fc0fcf24.diff
LOG: [SCEV] Return a SCEVUse from getMulExpr and propagate use flags. (#221486)
Same as https://github.com/llvm/llvm-project/pull/220007 but for
getMulExpr:
add option to pass SCEVUse-specific flags to getMulExpr and propagate
them through, if valid conservatively. That is, the final expression
multiples the same operands (potentially in different order). For
example, it is not valid to propagate the use flags if other
(sub-)expressions have been inlined.
It also includes a few mechanical changes, to update users that still
expected const SCEV *.
PR: https://github.com/llvm/llvm-project/pull/221486
Added:
Modified:
llvm/include/llvm/Analysis/ScalarEvolution.h
llvm/lib/Analysis/ScalarEvolution.cpp
llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
llvm/unittests/Analysis/ScalarEvolutionTest.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 33ba3a47e365e..c70dc55e3bbda 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -776,18 +776,15 @@ class ScalarEvolution {
SmallVector<SCEVUse, 3> Ops = {Op0, Op1, Op2};
return getAddExpr(Ops, Flags, Depth);
}
- LLVM_ABI const SCEV *getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
- SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
- unsigned Depth = 0);
- const SCEV *getMulExpr(SCEVUse LHS, SCEVUse RHS,
- SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
- unsigned Depth = 0) {
+ LLVM_ABI SCEVUse getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
+ SCEVFlags Flags = {}, unsigned Depth = 0);
+ SCEVUse getMulExpr(SCEVUse LHS, SCEVUse RHS, SCEVFlags Flags = {},
+ unsigned Depth = 0) {
SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
return getMulExpr(Ops, Flags, Depth);
}
- const SCEV *getMulExpr(SCEVUse Op0, SCEVUse Op1, SCEVUse Op2,
- SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
- unsigned Depth = 0) {
+ SCEVUse getMulExpr(SCEVUse Op0, SCEVUse Op1, SCEVUse Op2,
+ SCEVFlags Flags = {}, unsigned Depth = 0) {
SmallVector<SCEVUse, 3> Ops = {Op0, Op1, Op2};
return getMulExpr(Ops, Flags, Depth);
}
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index a57c489b534f1..689a8741af983 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -3128,11 +3128,14 @@ static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
}
/// Get a canonical multiply expression, or something simpler if possible.
-const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
- SCEV::NoWrapFlags OrigFlags,
- unsigned Depth) {
+SCEVUse ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
+ SCEVFlags Flags, unsigned Depth) {
+ SCEV::NoWrapFlags OrigFlags = Flags.ExprFlags;
+ SCEV::NoWrapFlags UseFlags = Flags.UseFlags;
assert(OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
"only nuw or nsw allowed");
+ assert(UseFlags == maskFlags(UseFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
+ "only nuw or nsw allowed");
assert(!Ops.empty() && "Cannot get empty mul!");
if (Ops.size() == 1) return Ops[0];
#ifndef NDEBUG
@@ -3151,6 +3154,12 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
if (Folded)
return Folded;
+#ifndef NDEBUG
+ // Keep track of operands after constant folding, for verification when adding
+ // use-specific flags.
+ const SmallVector<SCEVUse, 8> OrigOps(Ops.begin(), Ops.end());
+#endif
+
// Delay expensive flag strengthening until necessary.
auto ComputeFlags = [this, OrigFlags](const ArrayRef<SCEVUse> Ops) {
return StrengthenNoWrapFlags(this, scMulExpr, Ops, OrigFlags);
@@ -3158,14 +3167,14 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
// Limit recursion calls depth.
if (Depth > MaxArithDepth || hasHugeExpression(Ops))
- return getOrCreateMulExpr(Ops, ComputeFlags(Ops));
+ return {getOrCreateMulExpr(Ops, ComputeFlags(Ops)), UseFlags};
if (SCEV *S = findExistingSCEVInCache(scMulExpr, Ops)) {
// Don't strengthen flags if we have no new information.
SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
if (Mul->getNoWrapFlags(OrigFlags) != OrigFlags)
Mul->setNoWrapFlags(ComputeFlags(Ops));
- return S;
+ return {S, UseFlags};
}
if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
@@ -3232,7 +3241,8 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
hasFlags(StrengthenNoWrapFlags(this, scMulExpr, {NarrowC, InnerAdd},
SCEV::FlagAnyWrap),
SCEV::FlagNUW)) {
- auto *Res = getMulExpr(NarrowC, InnerAdd, SCEV::FlagNUW, Depth + 1);
+ const SCEV *Res =
+ getMulExpr(NarrowC, InnerAdd, SCEV::FlagNUW, Depth + 1);
return getZeroExtendExpr(Res, Ops[1]->getType(), Depth + 1);
};
}
@@ -3428,7 +3438,9 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
// Okay, it looks like we really DO need an mul expr. Check to see if we
// already have one, otherwise create a new one.
- return getOrCreateMulExpr(Ops, ComputeFlags(Ops));
+ assert((UseFlags == SCEV::FlagAnyWrap || equal(OrigOps, Ops)) &&
+ "Tried to add SCEVUse flags after operands changed");
+ return {getOrCreateMulExpr(Ops, ComputeFlags(Ops)), UseFlags};
}
/// Represents an unsigned remainder expression based on unsigned division.
@@ -7998,7 +8010,7 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
SmallVector<SCEVUse, 4> MulOps;
MulOps.push_back(getConstant(OpC->getAPInt().ashr(GCD)));
append_range(MulOps, LHSMul->operands().drop_front());
- auto *NewMul = getMulExpr(MulOps, LHSMul->getNoWrapFlags());
+ const SCEV *NewMul = getMulExpr(MulOps, LHSMul->getNoWrapFlags());
ShiftedLHS = getUDivExpr(NewMul, getConstant(DivAmt));
}
}
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 0528f9877804a..08a4a2f95033e 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -3916,7 +3916,7 @@ static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
for (const SCEV *S : Add->operands()) {
const SCEV *Remainder = CollectSubexprs(S, C, Ops, L, SE, Depth+1);
if (Remainder)
- Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder);
+ Ops.push_back(C ? SE.getMulExpr(C, Remainder).getPointer() : Remainder);
}
return nullptr;
}
@@ -3933,7 +3933,7 @@ static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
// does not pertain to this loop.
if (Remainder && (cast<SCEVAddRecExpr>(S)->getLoop() == L ||
!isa<SCEVAddRecExpr>(Remainder))) {
- Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder);
+ Ops.push_back(C ? SE.getMulExpr(C, Remainder).getPointer() : Remainder);
Remainder = nullptr;
}
if (Remainder != Start) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index ddde0cd83a704..3747700f75d54 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -776,7 +776,7 @@ void VPlanTransforms::materializeConstantVectorTripCount(
if (!isa<SCEVConstant>(TCScev))
return;
const SCEV *VFxUF = SE.getElementCount(TCScev->getType(), BestVF * BestUF);
- auto VecTCScev = SE.getMulExpr(SE.getUDivExpr(TCScev, VFxUF), VFxUF);
+ const SCEV *VecTCScev = SE.getMulExpr(SE.getUDivExpr(TCScev, VFxUF), VFxUF);
if (auto *ConstVecTC = dyn_cast<SCEVConstant>(VecTCScev))
Plan.getVectorTripCount().setUnderlyingValue(ConstVecTC->getValue());
}
diff --git a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
index e3fa7dd9c92b5..b45ad57414214 100644
--- a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
+++ b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
@@ -2458,4 +2458,103 @@ TEST_F(ScalarEvolutionsTest, AddExprUseFlags) {
#endif
});
}
+
+TEST_F(ScalarEvolutionsTest, MulExprUseFlags) {
+ LLVMContext C;
+ SMDiagnostic Err;
+ std::unique_ptr<Module> M = parseAssemblyString(
+ R"(define void @f(i32 %a, i32 %b, i32 %c) {
+ entry:
+ br label %loop
+
+ loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %iv.next = add i32 %iv, 1
+ %cond = icmp ult i32 %iv.next, 10
+ br i1 %cond, label %loop, label %exit
+
+ exit:
+ ret void
+ })",
+ Err, C);
+
+ if (!M) {
+ Err.print("ScalarEvolutionTest", errs());
+ ASSERT_TRUE(M && "Could not parse module?");
+ }
+ ASSERT_TRUE(!verifyModule(*M, &errs()) && "Must have been well formed!");
+
+ runWithSE(*M, "f", [](Function &F, LoopInfo &LI, ScalarEvolution &SE) {
+ const SCEV *A = SE.getSCEV(getArgByName(F, "a"));
+ const SCEV *B = SE.getSCEV(getArgByName(F, "b"));
+ const SCEV *Cc = SE.getSCEV(getArgByName(F, "c"));
+ const SCEV *IV = SE.getSCEV(getInstructionByName(F, "iv"));
+ ASSERT_TRUE(isa<SCEVAddRecExpr>(IV));
+ Type *I32 = A->getType();
+ const SCEV *Two = SE.getConstant(I32, 2);
+ const SCEV *Three = SE.getConstant(I32, 3);
+
+ // The product is built as-is, use carries the requested flags.
+ SCEVUse Prod = SE.getMulExpr(A, B, {SCEV::FlagAnyWrap, SCEV::FlagNUW});
+ EXPECT_TRUE(Prod.hasUseFlags());
+ EXPECT_EQ(Prod.getUseNoWrapFlags(), SCEV::FlagNUW | SCEV::FlagNW);
+
+ // Same as Prod, but without NUW use flags.
+ const SCEV *BareProd = SE.getMulExpr(A, B);
+ EXPECT_EQ(Prod.getPointer(), BareProd);
+ EXPECT_EQ(cast<SCEVMulExpr>(BareProd)->getNoWrapFlags(SCEV::FlagNUW),
+ SCEV::FlagAnyWrap);
+ EXPECT_EQ(Prod.getCanonical(), BareProd);
+ EXPECT_FALSE(SCEVUse(BareProd).hasUseFlags());
+
+ // Operands get sorted by complexity, so their order does not matter.
+ EXPECT_EQ(SE.getMulExpr(B, A, {SCEV::FlagAnyWrap, SCEV::FlagNUW}), Prod);
+
+ // The same holds for products of more than two operands.
+ SmallVector<SCEVUse, 3> Ops = {Cc, B, A};
+ SCEVUse Prod3 = SE.getMulExpr(Ops, {SCEV::FlagAnyWrap, SCEV::FlagNSW});
+ EXPECT_EQ(Prod3.getUseNoWrapFlags(), SCEV::FlagNSW | SCEV::FlagNW);
+ EXPECT_EQ(Prod3.getCanonical(), SE.getMulExpr(A, B, Cc));
+
+ // Flags the expression already carries add nothing to the use.
+ SCEVUse NUWProd = SE.getMulExpr(A, Cc, {SCEV::FlagNUW, SCEV::FlagNUW});
+ ASSERT_TRUE(cast<SCEVMulExpr>(NUWProd.getPointer())->hasNoUnsignedWrap());
+ EXPECT_FALSE(NUWProd.hasUseFlags());
+
+ // A product folded to a single operand, a flattened nested product and a
+ // constant distributed over a sum or into a recurrence all describe a
+ //
diff erent computation than the requested product, so none of them may
+ // carry its flags.
+ auto CheckNoUseFlags = [](SCEVUse U) {
+ EXPECT_FALSE(U.hasUseFlags());
+ EXPECT_EQ(U.getUseNoWrapFlags(), SCEV::FlagAnyWrap);
+ };
+ CheckNoUseFlags(
+ SE.getMulExpr(Two, Three, {SCEV::FlagAnyWrap, SCEV::FlagNUW}));
+ CheckNoUseFlags(
+ SE.getMulExpr(A, SE.getOne(I32), {SCEV::FlagAnyWrap, SCEV::FlagNUW}));
+ CheckNoUseFlags(
+ SE.getMulExpr(A, SE.getZero(I32), {SCEV::FlagAnyWrap, SCEV::FlagNUW}));
+ CheckNoUseFlags(SE.getMulExpr(A, SE.getMulExpr(B, Cc),
+ {SCEV::FlagAnyWrap, SCEV::FlagNUW}));
+ CheckNoUseFlags(SE.getMulExpr(Two, SE.getAddExpr(SE.getOne(I32), A),
+ {SCEV::FlagAnyWrap, SCEV::FlagNUW}));
+ CheckNoUseFlags(SE.getMulExpr(Two, IV, {SCEV::FlagAnyWrap, SCEV::FlagNUW}));
+
+ // SCEV flags are valid for all subsets and orders of the operands, so
+ // use-specific flags can be preserved when folding constants.
+ SmallVector<SCEVUse, 3> FoldedOps = {Two, Three, A};
+ SCEVUse FoldedProd = SE.getMulExpr(
+ FoldedOps, {SCEV::FlagAnyWrap, SCEV::FlagNUW | SCEV::FlagNSW});
+ EXPECT_EQ(FoldedProd.getCanonical(),
+ SE.getMulExpr(SE.getConstant(I32, 6), A));
+ EXPECT_EQ(FoldedProd.getUseNoWrapFlags(),
+ SCEV::FlagNUW | SCEV::FlagNSW | SCEV::FlagNW);
+
+#ifndef NDEBUG
+ EXPECT_DEATH((void)SE.getMulExpr(A, B, {SCEV::FlagAnyWrap, SCEV::FlagNW}),
+ "only nuw or nsw allowed");
+#endif
+ });
+}
} // end namespace llvm
More information about the llvm-commits
mailing list