[llvm] [SCEV] Introduce UDiv::mayTriggerUB (PR #217064)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 08:57:34 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Ramkumar Ramachandra (artagnon)
<details>
<summary>Changes</summary>
Use it to clean up the logic in SCEVExpander, leading to one improvement in LoopVectorize. While at it, make a related non-functional change in ScalarEvolution.
---
Patch is 60.57 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/217064.diff
4 Files Affected:
- (modified) llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h (+5)
- (modified) llvm/lib/Analysis/ScalarEvolution.cpp (+132-133)
- (modified) llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp (+8-18)
- (modified) llvm/test/Transforms/LoopVectorize/pr38697.ll (+575-40)
``````````diff
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
index ebee63963c701..6aea576c181cb 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -319,6 +319,11 @@ class SCEVUDivExpr : public SCEV {
/// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const SCEV *S) { return S->getSCEVType() == scUDivExpr; }
+
+ /// Returns true if the expression may trigger undefined-behavior.
+ bool mayTriggerUB(ScalarEvolution &SE) const {
+ return !SE.isKnownNonZero(getRHS());
+ }
};
/// This node represents a polynomial recurrence on the trip count
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 1707c6c17fe23..cd3c3e6bba777 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -3514,161 +3514,160 @@ const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
return LHS;
if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
- if (RHSC->getValue()->isOne())
- return LHS; // X udiv 1 --> x
// If the denominator is zero, the result of the udiv is undefined. Don't
// try to analyze it, because the resolution chosen here may differ from
// the resolution chosen in other parts of the compiler.
- if (!RHSC->getValue()->isZero()) {
- // Determine if the division can be folded into the operands of
- // its operands.
- // TODO: Generalize this to non-constants by using known-bits information.
- Type *Ty = LHS->getType();
- unsigned LZ = RHSC->getAPInt().countl_zero();
- unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
- // For non-power-of-two values, effectively round the value up to the
- // nearest power of two.
- if (!RHSC->getAPInt().isPowerOf2())
- ++MaxShiftAmt;
- IntegerType *ExtTy =
+ if (RHSC->getValue()->isZero())
+ return getOrCreateUDivExpr(LHS, RHS);
+
+ if (RHSC->getValue()->isOne())
+ return LHS; // X udiv 1 --> x
+
+ // Determine if the division can be folded into the operands of
+ // its operands.
+ // TODO: Generalize this to non-constants by using known-bits information.
+ Type *Ty = LHS->getType();
+ unsigned LZ = RHSC->getAPInt().countl_zero();
+ unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
+ // For non-power-of-two values, effectively round the value up to the
+ // nearest power of two.
+ if (!RHSC->getAPInt().isPowerOf2())
+ ++MaxShiftAmt;
+ IntegerType *ExtTy =
IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt);
- if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
- if (const SCEVConstant *Step =
- dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) {
- // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
- const APInt &StepInt = Step->getAPInt();
- const APInt &DivInt = RHSC->getAPInt();
- if (!StepInt.urem(DivInt) &&
- getZeroExtendExpr(AR, ExtTy) ==
- getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
- getZeroExtendExpr(Step, ExtTy),
- AR->getLoop(), SCEV::FlagAnyWrap)) {
- SmallVector<SCEVUse, 4> Operands;
- for (const SCEV *Op : AR->operands())
- Operands.push_back(getUDivExpr(Op, RHS));
- return getAddRecExpr(Operands, AR->getLoop(), SCEV::FlagNW);
- }
- /// Get a canonical UDivExpr for a recurrence.
- /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
- const APInt *StartRem;
- if (!DivInt.urem(StepInt) && match(getURemExpr(AR->getStart(), Step),
- m_scev_APInt(StartRem))) {
- bool NoWrap =
- getZeroExtendExpr(AR, ExtTy) ==
+ if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
+ if (const SCEVConstant *Step =
+ dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) {
+ // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
+ const APInt &StepInt = Step->getAPInt();
+ const APInt &DivInt = RHSC->getAPInt();
+ if (!StepInt.urem(DivInt) &&
+ getZeroExtendExpr(AR, ExtTy) ==
getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
getZeroExtendExpr(Step, ExtTy), AR->getLoop(),
- SCEV::FlagAnyWrap);
-
- // With N <= C and both N, C as powers-of-2, the transformation
- // {X,+,N}/C => {(X - X%N),+,N}/C preserves division results even
- // if wrapping occurs, as the division results remain equivalent for
- // all offsets in [[(X - X%N), X).
- bool CanFoldWithWrap = StepInt.ule(DivInt) && // N <= C
- StepInt.isPowerOf2() && DivInt.isPowerOf2();
- // Only fold if the subtraction can be folded in the start
- // expression.
- const SCEV *NewStart =
- getMinusSCEV(AR->getStart(), getConstant(*StartRem));
- if (*StartRem != 0 && (NoWrap || CanFoldWithWrap) &&
- !isa<SCEVAddExpr>(NewStart)) {
- const SCEV *NewLHS =
- getAddRecExpr(NewStart, Step, AR->getLoop(),
- NoWrap ? SCEV::FlagNW : SCEV::FlagAnyWrap);
- if (LHS != NewLHS)
- return getUDivExpr(NewLHS, RHS);
- }
- }
+ SCEV::FlagAnyWrap)) {
+ SmallVector<SCEVUse, 4> Operands;
+ for (const SCEV *Op : AR->operands())
+ Operands.push_back(getUDivExpr(Op, RHS));
+ return getAddRecExpr(Operands, AR->getLoop(), SCEV::FlagNW);
}
- // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
- if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
- SmallVector<SCEVUse, 4> Operands;
- for (const SCEV *Op : M->operands())
- Operands.push_back(getZeroExtendExpr(Op, ExtTy));
- if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands)) {
- // Find an operand that's safely divisible.
- for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
- const SCEV *Op = M->getOperand(i);
- const SCEV *Div = getUDivExpr(Op, RHSC);
- if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) {
- Operands = SmallVector<SCEVUse, 4>(M->operands());
- Operands[i] = Div;
- return getMulExpr(Operands);
- }
- }
-
- // Even if it's not divisible, try to remove a common factor.
- if (const auto *LHSC = dyn_cast<SCEVConstant>(M->getOperand(0))) {
- APInt Factor = APIntOps::GreatestCommonDivisor(LHSC->getAPInt(),
- RHSC->getAPInt());
- if (!Factor.isIntN(1)) {
- SmallVector<SCEVUse, 2> NewOperands;
- NewOperands.push_back(getConstant(LHSC->getAPInt().udiv(Factor)));
- append_range(NewOperands, M->operands().drop_front());
- const SCEV *NewMul = getMulExpr(NewOperands);
- return getUDivExpr(NewMul,
- getConstant(RHSC->getAPInt().udiv(Factor)));
- }
+ /// Get a canonical UDivExpr for a recurrence.
+ /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
+ const APInt *StartRem;
+ if (!DivInt.urem(StepInt) &&
+ match(getURemExpr(AR->getStart(), Step), m_scev_APInt(StartRem))) {
+ bool NoWrap = getZeroExtendExpr(AR, ExtTy) ==
+ getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
+ getZeroExtendExpr(Step, ExtTy),
+ AR->getLoop(), SCEV::FlagAnyWrap);
+
+ // With N <= C and both N, C as powers-of-2, the transformation
+ // {X,+,N}/C => {(X - X%N),+,N}/C preserves division results even
+ // if wrapping occurs, as the division results remain equivalent for
+ // all offsets in [[(X - X%N), X).
+ bool CanFoldWithWrap = StepInt.ule(DivInt) && // N <= C
+ StepInt.isPowerOf2() && DivInt.isPowerOf2();
+ // Only fold if the subtraction can be folded in the start
+ // expression.
+ const SCEV *NewStart =
+ getMinusSCEV(AR->getStart(), getConstant(*StartRem));
+ if (*StartRem != 0 && (NoWrap || CanFoldWithWrap) &&
+ !isa<SCEVAddExpr>(NewStart)) {
+ const SCEV *NewLHS =
+ getAddRecExpr(NewStart, Step, AR->getLoop(),
+ NoWrap ? SCEV::FlagNW : SCEV::FlagAnyWrap);
+ if (LHS != NewLHS)
+ return getUDivExpr(NewLHS, RHS);
}
}
}
+ // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
+ if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
+ SmallVector<SCEVUse, 4> Operands;
+ for (const SCEV *Op : M->operands())
+ Operands.push_back(getZeroExtendExpr(Op, ExtTy));
+ if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands)) {
+ // Find an operand that's safely divisible.
+ for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
+ const SCEV *Op = M->getOperand(i);
+ const SCEV *Div = getUDivExpr(Op, RHSC);
+ if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) {
+ Operands = SmallVector<SCEVUse, 4>(M->operands());
+ Operands[i] = Div;
+ return getMulExpr(Operands);
+ }
+ }
- // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
- if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(LHS)) {
- if (auto *DivisorConstant =
- dyn_cast<SCEVConstant>(OtherDiv->getRHS())) {
- bool Overflow = false;
- APInt NewRHS =
- DivisorConstant->getAPInt().umul_ov(RHSC->getAPInt(), Overflow);
- if (Overflow) {
- return getConstant(RHSC->getType(), 0, false);
+ // Even if it's not divisible, try to remove a common factor.
+ if (const auto *LHSC = dyn_cast<SCEVConstant>(M->getOperand(0))) {
+ APInt Factor = APIntOps::GreatestCommonDivisor(LHSC->getAPInt(),
+ RHSC->getAPInt());
+ if (!Factor.isIntN(1)) {
+ SmallVector<SCEVUse, 2> NewOperands;
+ NewOperands.push_back(getConstant(LHSC->getAPInt().udiv(Factor)));
+ append_range(NewOperands, M->operands().drop_front());
+ const SCEV *NewMul = getMulExpr(NewOperands);
+ return getUDivExpr(NewMul,
+ getConstant(RHSC->getAPInt().udiv(Factor)));
}
- return getUDivExpr(OtherDiv->getLHS(), getConstant(NewRHS));
}
}
+ }
- // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded.
- if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) {
- SmallVector<SCEVUse, 4> Operands;
- for (const SCEV *Op : A->operands())
- Operands.push_back(getZeroExtendExpr(Op, ExtTy));
- if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) {
- Operands.clear();
- for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
- const SCEV *Op = getUDivExpr(A->getOperand(i), RHS);
- if (isa<SCEVUDivExpr>(Op) ||
- getMulExpr(Op, RHS) != A->getOperand(i))
- break;
- Operands.push_back(Op);
- }
- if (Operands.size() == A->getNumOperands())
- return getAddExpr(Operands);
+ // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
+ if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(LHS)) {
+ if (auto *DivisorConstant = dyn_cast<SCEVConstant>(OtherDiv->getRHS())) {
+ bool Overflow = false;
+ APInt NewRHS =
+ DivisorConstant->getAPInt().umul_ov(RHSC->getAPInt(), Overflow);
+ if (Overflow) {
+ return getConstant(RHSC->getType(), 0, false);
}
+ return getUDivExpr(OtherDiv->getLHS(), getConstant(NewRHS));
}
+ }
- // ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
- // This is an idiom for rounding A up to the next multiple of N, where A
- // is aready known to be a multiple of M. In this case, instcombine can
- // see that some low bits of the added constant are unused, so can clear
- // them, but we want to canonicalise to set the low bits. This makes the
- // pattern easier to match, without needing to check for known bits in
- // A*M.
- const APInt &N = RHSC->getAPInt();
- const APInt *NMinusM, *M;
- const SCEV *A;
- if (match(LHS, m_scev_Add(m_scev_APInt(NMinusM),
- m_scev_Mul(m_scev_APInt(M), m_SCEV(A))))) {
- if (N.isPowerOf2() && M->isPowerOf2() && M->ult(N) &&
- *NMinusM == N - *M) {
- return getUDivExpr(
- getAddExpr(getConstant(N - 1), getMulExpr(getConstant(*M), A)),
- RHS);
+ // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded.
+ if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) {
+ SmallVector<SCEVUse, 4> Operands;
+ for (const SCEV *Op : A->operands())
+ Operands.push_back(getZeroExtendExpr(Op, ExtTy));
+ if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) {
+ Operands.clear();
+ for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
+ const SCEV *Op = getUDivExpr(A->getOperand(i), RHS);
+ if (isa<SCEVUDivExpr>(Op) || getMulExpr(Op, RHS) != A->getOperand(i))
+ break;
+ Operands.push_back(Op);
}
+ if (Operands.size() == A->getNumOperands())
+ return getAddExpr(Operands);
}
+ }
- // Fold if both operands are constant.
- if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS))
- return getConstant(LHSC->getAPInt().udiv(RHSC->getAPInt()));
+ // ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
+ // This is an idiom for rounding A up to the next multiple of N, where A
+ // is aready known to be a multiple of M. In this case, instcombine can
+ // see that some low bits of the added constant are unused, so can clear
+ // them, but we want to canonicalise to set the low bits. This makes the
+ // pattern easier to match, without needing to check for known bits in
+ // A*M.
+ const APInt &N = RHSC->getAPInt();
+ const APInt *NMinusM, *M;
+ const SCEV *A;
+ if (match(LHS, m_scev_Add(m_scev_APInt(NMinusM),
+ m_scev_Mul(m_scev_APInt(M), m_SCEV(A))))) {
+ if (N.isPowerOf2() && M->isPowerOf2() && M->ult(N) &&
+ *NMinusM == N - *M) {
+ return getUDivExpr(
+ getAddExpr(getConstant(N - 1), getMulExpr(getConstant(*M), A)),
+ RHS);
+ }
}
+
+ // Fold if both operands are constant.
+ if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS))
+ return getConstant(LHSC->getAPInt().udiv(RHSC->getAPInt()));
}
// ((-C + (C smax %x)) /u %x) evaluates to zero, for any positive constant C.
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index a7ca2104e59c3..e7879d8b09a55 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -725,12 +725,12 @@ Value *SCEVExpander::visitUDivExpr(SCEVUseT<const SCEVUDivExpr *> S) {
// We need an umax if either RHSExpr is not known to be zero, or if it is
// not guaranteed to be non-poison. In the later case, the frozen poison may
// be 0.
- if (!SE.isKnownNonZero(RHSExpr) || !GuaranteedNotPoison)
+ if (S->mayTriggerUB(SE) || !GuaranteedNotPoison)
RHS = Builder.CreateIntrinsic(RHS->getType(), Intrinsic::umax,
{RHS, ConstantInt::get(RHS->getType(), 1)});
}
return InsertBinop(Instruction::UDiv, LHS, RHS, SCEV::FlagAnyWrap,
- /*IsSafeToHoist*/ SE.isKnownNonZero(S->getRHS()));
+ /*IsSafeToHoist=*/!S->mayTriggerUB(SE));
}
/// Determine if this is a well-behaved chain of instructions leading back to
@@ -1667,20 +1667,11 @@ Value *SCEVExpander::expand(SCEVUse S) {
// We can move insertion point only if there is no div or rem operations
// otherwise we are risky to move it over the check for zero denominator.
- auto SafeToHoist = [](const SCEV *S) {
- return !SCEVExprContains(S, [](const SCEV *S) {
- if (const auto *D = dyn_cast<SCEVUDivExpr>(S)) {
- if (const auto *SC = dyn_cast<SCEVConstant>(D->getRHS()))
- // Division by non-zero constants can be hoisted.
- return SC->getValue()->isZero();
- // All other divisions should not be moved as they may be
- // divisions by zero and should be kept within the
- // conditions of the surrounding loops that guard their
- // execution (see PR35406).
- return true;
- }
- return false;
- });
+ auto SafeToHoist = [this](const SCEV *S) {
+ return !SCEVExprContains(S, [this](const SCEV *S) {
+ const auto *D = dyn_cast<SCEVUDivExpr>(S);
+ return D && D->mayTriggerUB(SE);
+ });
};
if (SafeToHoist(S)) {
for (Loop *L = SE.LI.getLoopFor(Builder.GetInsertBlock());;
@@ -2507,8 +2498,7 @@ struct SCEVFindUnsafe {
bool follow(const SCEV *S) {
if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) {
- if (!SE.isKnownNonZero(D->getRHS()) ||
- !SE.isGuaranteedNotToBePoison(D->getRHS())) {
+ if (D->mayTriggerUB(SE) || !SE.isGuaranteedNotToBePoison(D->getRHS())) {
IsUnsafe = true;
return false;
}
diff --git a/llvm/test/Transforms/LoopVectorize/pr38697.ll b/llvm/test/Transforms/LoopVectorize/pr38697.ll
index 5570a1d8f7bd0..7caa7a99b509b 100644
--- a/llvm/test/Transforms/LoopVectorize/pr38697.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr38697.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=2 -S < %s 2>&1 | FileCheck %s
; RUN: opt -passes=indvars -S < %s 2>&1 | FileCheck %s -check-prefix=INDVARCHECK
@@ -31,17 +32,114 @@ target triple = "x86_64-unknown-linux-gnu"
; Verify that a 'udiv' does not appear in the 'loop1.preheader' block, and that
; a 'udiv' has been inserted at the top of the 'while.body.preheader' block.
define void @testCountIncrLoop(ptr %ptr, i32 %lim, i32 %count, i32 %val) mustprogress {
-; CHECK-LABEL: @testCountIncrLoop(
-; CHECK-NEXT: entry:
-; CHECK: loop1.preheader:
-; CHECK-NOT: udiv
-; CHECK: loop1.body:
-; CHECK: while.cond.preheader:
-; CHECK: while.body.preheader:
-; CHECK: [[TMP1:%.*]] = udiv i32 [[TMP0:%.*]], [[COUNT:%.*]]
-; CHECK: vector.ph:
-; CHECK: exit:
-; CHECK: ret void
+; CHECK-LABEL: define void @testCountIncrLoop(
+; CHECK-SAME: ptr [[PTR:%.*]], i32 [[LIM:%.*]], i32 [[COUNT:%.*]], i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[LIM]], 0
+; CHECK-NEXT: br i1 [[CMP1]], label %[[LOOP1_PREHEADER:.*]], [[EXIT:label %.*]]
+; CHECK: [[LOOP1_PREHEADER]]:
+; CHECK-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[COUNT]], 0
+; CHECK-NEXT: [[CMP4:%.*]] = icmp slt i32 [[COUNT]], 8
+; CHECK-NEXT: br label %[[LOOP1_BODY:.*]]
+; CHECK: [[LOOP1_BODY]]:
+; CHECK-NEXT: [[OUTER_I:%.*]] = phi i32 [ 0, %[[LOOP1_PREHEADER]] ], [ [[OUTER_I_1:%.*]], %[[LOOP1_INC:.*]] ]
+; CHECK-NEXT: [[INX_1:%.*]] = phi i32 [ 0, %[[LOOP1_PREHEADER]] ], [ [[INX_2:%.*]], %[[LOOP1_INC]] ]
+; CHECK-NEXT: br i1 [[CMP2]], label %[[WHILE_COND_PREHEADER:.*]], label %[[LOOP1_INC]]
+; ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/217064
More information about the llvm-commits
mailing list