[llvm] [SCEV] Introduce exact flag for UDiv (PR #225065)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 01:46:33 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/225065
>From cc25c36577006e361f94cbb1748ec4aedb50f1b1 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 21 Sep 2026 13:17:08 +0100
Subject: [PATCH 1/3] [LoopUnroll] Pre-commit udiv-exact tests
---
.../LoopUnroll/runtime-udiv-exact.ll | 123 ++++++++++++++++++
1 file changed, 123 insertions(+)
create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-udiv-exact.ll
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-udiv-exact.ll b/llvm/test/Transforms/LoopUnroll/runtime-udiv-exact.ll
new file mode 100644
index 0000000000000..ef7ad8c6b9c32
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-udiv-exact.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-unroll -unroll-runtime -unroll-count=2 -S %s | FileCheck %s
+
+; The trip count is (%n /u 24)<exact>. The udiv computing it is inside the
+; loop, so SCEV expander must emit a fresh one in the preheader: it should carry
+; the exact flag.
+define void @expand_exact_udiv(i64 %n, ptr %p) {
+; CHECK-LABEL: define void @expand_exact_udiv(
+; CHECK-SAME: i64 [[N:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 [[N]], 24
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], -1
+; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i64 [[TMP1]], 1
+; CHECK-NEXT: br i1 [[TMP2]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
+; CHECK: [[ENTRY_NEW]]:
+; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP0]], [[XTRAITER]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
+; CHECK-NEXT: store i64 [[IV]], ptr [[GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
+; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV_NEXT]]
+; CHECK-NEXT: store i64 [[IV_NEXT]], ptr [[GEP_1]], align 4
+; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp ne i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[LOOP]], label %[[EXIT_UNR_LCSSA:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT_UNR_LCSSA]]:
+; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_EPIL_PREHEADER]]:
+; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]])
+; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
+; CHECK: [[LOOP_EPIL]]:
+; CHECK-NEXT: [[GEP_EPIL:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV_EPIL_INIT]]
+; CHECK-NEXT: store i64 [[IV_EPIL_INIT]], ptr [[GEP_EPIL]], align 4
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %gep = getelementptr i64, ptr %p, i64 %iv
+ store i64 %iv, ptr %gep
+ %q = udiv exact i64 %n, 24
+ %ec = icmp ne i64 %iv.next, %q
+ br i1 %ec, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Same, but an identical exact udiv already exists in the preheader. SCEV
+; expander should reuse it instead of emitting a duplicate.
+define void @reuse_exact_udiv(i64 %n, ptr %p, ptr %q.out) {
+; CHECK-LABEL: define void @reuse_exact_udiv(
+; CHECK-SAME: i64 [[N:%.*]], ptr [[P:%.*]], ptr [[Q_OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[Q_PH:%.*]] = udiv exact i64 [[N]], 24
+; CHECK-NEXT: store i64 [[Q_PH]], ptr [[Q_OUT]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 [[N]], 24
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[TMP2]], -1
+; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP2]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
+; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
+; CHECK: [[ENTRY_NEW]]:
+; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP2]], [[XTRAITER]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
+; CHECK-NEXT: store i64 [[IV]], ptr [[GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
+; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV_NEXT]]
+; CHECK-NEXT: store i64 [[IV_NEXT]], ptr [[GEP_1]], align 4
+; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp ne i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[LOOP]], label %[[EXIT_UNR_LCSSA:.*]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[EXIT_UNR_LCSSA]]:
+; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_EPIL_PREHEADER]]:
+; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]])
+; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
+; CHECK: [[LOOP_EPIL]]:
+; CHECK-NEXT: [[GEP_EPIL:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV_EPIL_INIT]]
+; CHECK-NEXT: store i64 [[IV_EPIL_INIT]], ptr [[GEP_EPIL]], align 4
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %q.ph = udiv exact i64 %n, 24
+ store i64 %q.ph, ptr %q.out
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %gep = getelementptr i64, ptr %p, i64 %iv
+ store i64 %iv, ptr %gep
+ %q = udiv exact i64 %n, 24
+ %ec = icmp ne i64 %iv.next, %q
+ br i1 %ec, label %loop, label %exit
+
+exit:
+ ret void
+}
>From c57ea523161cac62b942681cab15e87bba18ef45 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 21 Sep 2026 12:29:53 +0100
Subject: [PATCH 2/3] [SCEV] Introduce exact flag for UDiv
Introduce SCEVFlags::FlagExact for UDiv expressions. This allows us to
model PossiblyExactOperators in IR faithfully, and complete
getUDivExactExpr. The resulting improvements from SCEV not dropping
exact flags are free.
Assisted-by: llm, for testing
---
llvm/include/llvm/Analysis/ScalarEvolution.h | 27 +++++-----
.../Analysis/ScalarEvolutionExpressions.h | 53 ++++++++++++++-----
llvm/lib/Analysis/ScalarEvolution.cpp | 46 +++++++++-------
.../Utils/ScalarEvolutionExpander.cpp | 12 +++--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +-
.../runtime-check-known-true.ll | 4 +-
.../ScalarEvolution/flags-from-poison.ll | 2 +-
.../Analysis/ScalarEvolution/ne-overflow.ll | 40 +++++++-------
.../LoopUnroll/runtime-udiv-exact.ll | 9 ++--
.../Analysis/ScalarEvolutionTest.cpp | 10 +++-
10 files changed, 125 insertions(+), 80 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 4d9f0247ef640..5753f00800936 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -113,7 +113,8 @@ enum class SCEVFlags {
FlagNUW = (1 << 1), // No unsigned wrap.
FlagNSW = (1 << 2), // No signed wrap.
FlagsNoWrapMask = (1 << 3) - 1,
- FlagsMask = (1 << 3) - 1,
+ FlagExact = (1 << 3), // Exact flag.
+ FlagsMask = (1 << 4) - 1,
LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/FlagsMask)
};
@@ -135,7 +136,7 @@ struct SCEVUseT : private PointerIntPair<SCEVPtrT, 2> {
template <typename OtherPtrT, typename = std::enable_if_t<
std::is_convertible_v<OtherPtrT, SCEVPtrT>>>
SCEVUseT(const SCEVUseT<OtherPtrT> &Other)
- : SCEVUseT(Other.getPointer(), Other.getUseNoWrapFlags()) {}
+ : SCEVUseT(Other.getPointer(), Other.getUseFlags()) {}
operator SCEVPtrT() const { return getPointer(); }
SCEVPtrT operator->() const { return getPointer(); }
@@ -154,16 +155,14 @@ struct SCEVUseT : private PointerIntPair<SCEVPtrT, 2> {
/// flags and the underlying SCEV's flags, masked by \p Mask.
SCEVFlags getNoWrapFlags(SCEVFlags Mask = SCEVFlags::FlagsNoWrapMask) const;
+ /// Return the flags for this SCEVUse, masked for the exact flag.
+ SCEVFlags getExactFlag() const;
+
/// Return only the use-specific flags without the underlying SCEV's flags.
SCEVFlags getUseNoWrapFlags() const {
return getUseFlags() & SCEVFlags::FlagsNoWrapMask;
}
- SCEVFlags getUseFlags() const {
- SCEVFlags UseFlags = static_cast<SCEVFlags>(Base::getInt() << 1);
- if (any(UseFlags & (SCEVFlags::FlagNUW | SCEVFlags::FlagNSW)))
- UseFlags |= SCEVFlags::FlagNW;
- return UseFlags;
- }
+ SCEVFlags getUseFlags() const;
bool operator==(const SCEVUseT &RHS) const {
return getOpaqueValue() == RHS.getOpaqueValue();
@@ -250,7 +249,7 @@ struct CastInfo<SCEVUseT<ToSCEVPtrT>, SCEVUse,
static bool isPossible(const SCEVUse &U) { return isa<To>(U.getPointer()); }
static CastReturnType doCast(const SCEVUse &U) {
- return CastReturnType(cast<To>(U.getPointer()), U.getUseNoWrapFlags());
+ return CastReturnType(cast<To>(U.getPointer()), U.getUseFlags());
}
static CastReturnType castFailed() { return CastReturnType(nullptr); }
static CastReturnType doCastIfPossible(const SCEVUse &U) {
@@ -300,6 +299,7 @@ class SCEV : public FoldingSetNode {
static constexpr auto FlagNSW = SCEVFlags::FlagNSW;
static constexpr auto FlagsNoWrapMask = SCEVFlags::FlagsNoWrapMask;
static constexpr auto FlagsMask = SCEVFlags::FlagsMask;
+ static constexpr auto FlagExact = SCEVFlags::FlagExact;
explicit SCEV(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
unsigned short ExpressionSize, Type *Ty)
@@ -780,7 +780,8 @@ class ScalarEvolution {
SmallVector<SCEVUse, 3> Ops = {Op0, Op1, Op2};
return getMulExpr(Ops, Flags, Depth);
}
- LLVM_ABI const SCEV *getUDivExpr(SCEVUse LHS, SCEVUse RHS);
+ LLVM_ABI SCEVUse getUDivExpr(SCEVUse LHS, SCEVUse RHS,
+ SCEVFlagsPair Flags = {});
LLVM_ABI const SCEV *getUDivExactExpr(SCEVUse LHS, SCEVUse RHS);
LLVM_ABI const SCEV *getURemExpr(SCEVUse LHS, SCEVUse RHS);
LLVM_ABI SCEVUse getAddRecExpr(SCEVUse Start, SCEVUse Step, const Loop *L,
@@ -2561,7 +2562,7 @@ class ScalarEvolution {
SCEVFlags Flags);
// Get UDiv expression already created or create a new one.
- const SCEV *getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS);
+ const SCEV *getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS, SCEVFlags Flags);
/// Return x if \p Val is f(x) where f is a 1-1 function.
const SCEV *stripInjectiveFunctions(const SCEV *Val) const;
@@ -2797,11 +2798,13 @@ template <> inline const SCEV *SCEVUseT<const SCEV *>::getCanonical() const {
template <typename SCEVPtrT>
void SCEVUseT<SCEVPtrT>::print(raw_ostream &OS) const {
getPointer()->print(OS);
- SCEVFlags Flags = getUseNoWrapFlags();
+ SCEVFlags Flags = getUseFlags();
if (any(Flags & SCEV::FlagNUW))
OS << "<u nuw>";
if (any(Flags & SCEV::FlagNSW))
OS << "<u nsw>";
+ if (any(Flags & SCEV::FlagExact))
+ OS << "<u exact>";
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
index ae10884a4dba6..02c1c04190fad 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -246,11 +246,8 @@ class SCEVCommutativeExpr : public SCEVNAryExpr {
}
/// Set flags for a non-recurrence without clearing previously set flags.
- void setFlags(SCEVFlags Flags) {
- SubclassData |= static_cast<unsigned short>(Flags);
- }
void setNoWrapFlags(SCEVFlags Flags) {
- setFlags(Flags & SCEV::FlagsNoWrapMask);
+ SubclassData |= static_cast<unsigned short>(Flags & SCEV::FlagsNoWrapMask);
}
};
@@ -313,6 +310,13 @@ class SCEVUDivExpr : public SCEV {
ArrayRef<SCEVUse> operands() const { return Operands; }
+ void setExactFlag(SCEVFlags Flags) {
+ SubclassData |= static_cast<unsigned short>(Flags & SCEV::FlagExact);
+ }
+ SCEVFlags getExactFlag() const {
+ return static_cast<SCEVFlags>(SubclassData) & SCEV::FlagExact;
+ }
+
/// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const SCEV *S) { return S->getSCEVType() == scUDivExpr; }
};
@@ -508,11 +512,8 @@ class SCEVSequentialMinMaxExpr : public SCEVNAryExpr {
}
/// Set flags for a non-recurrence without clearing previously set flags.
- void setFlags(SCEVFlags Flags) {
- SubclassData |= static_cast<unsigned short>(Flags);
- }
void setNoWrapFlags(SCEVFlags Flags) {
- setFlags(Flags & SCEV::FlagsNoWrapMask);
+ SubclassData |= static_cast<unsigned short>(Flags & SCEV::FlagsNoWrapMask);
}
protected:
@@ -1005,15 +1006,21 @@ class SCEVLoopAddRecRewriter
template <typename SCEVPtrT>
inline SCEVUseT<SCEVPtrT>::SCEVUseT(SCEVPtrT S, SCEVFlags Flags) : Base(S, 0) {
- assert((Flags & SCEVFlags::FlagsNoWrapMask) == Flags &&
- "Expected only no-wrap flags");
- if (any(Flags)) {
+ if (any(Flags & SCEVFlags::FlagsNoWrapMask)) {
assert((isa<SCEVAddExpr, SCEVMulExpr, SCEVAddRecExpr>(S)) &&
- "use flags require an expression that can carry no-wrap flags");
- // Drop flags already present on S.
+ "use no-wrap flags require an expression that can carry one");
+ // Drop no-wrap flags already present on S.
Flags &= ~cast<SCEVNAryExpr>(S)->getNoWrapFlags();
+ Base::setInt(static_cast<unsigned>(Flags) >> 1);
+ return;
+ }
+ if (any(Flags & SCEVFlags::FlagExact)) {
+ assert(isa<SCEVUDivExpr>(S) &&
+ "use exact flag requires an expression that can carry one");
+ // Drop exact flags already present on S.
+ Flags &= ~cast<SCEVUDivExpr>(S)->getExactFlag();
+ Base::setInt(static_cast<unsigned>(Flags) >> 3);
}
- Base::setInt(static_cast<unsigned>(Flags) >> 1);
}
template <typename SCEVPtrT>
@@ -1024,6 +1031,24 @@ inline SCEVFlags SCEVUseT<SCEVPtrT>::getNoWrapFlags(SCEVFlags Mask) const {
return (Flags | getUseNoWrapFlags()) & Mask;
}
+template <>
+inline SCEVFlags SCEVUseT<const SCEVUDivExpr *>::getExactFlag() const {
+ SCEVFlags Flags = SCEVFlags::FlagNone;
+ if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Base::getPointer()))
+ Flags = UDiv->getExactFlag();
+ return (Flags | getUseFlags()) & SCEV::FlagExact;
+}
+
+template <typename SCEVPtrT>
+inline SCEVFlags SCEVUseT<SCEVPtrT>::getUseFlags() const {
+ if constexpr (std::is_convertible_v<SCEVPtrT, const SCEVUDivExpr *>)
+ return static_cast<SCEVFlags>(Base::getInt() << 3);
+ SCEVFlags UseFlags = static_cast<SCEVFlags>(Base::getInt() << 1);
+ if (any(UseFlags & (SCEVFlags::FlagNUW | SCEVFlags::FlagNSW)))
+ UseFlags |= SCEVFlags::FlagNW;
+ return UseFlags;
+}
+
} // end namespace llvm
#endif // LLVM_ANALYSIS_SCALAREVOLUTIONEXPRESSIONS_H
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 5b8e30985ee7f..07bed756ed44e 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -391,6 +391,8 @@ void SCEV::print(raw_ostream &OS) const {
case scUDivExpr: {
const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this);
OS << "(" << UDiv->getLHS() << " /u " << UDiv->getRHS() << ")";
+ if (any(UDiv->getExactFlag() & SCEV::FlagExact))
+ OS << "<exact>";
return;
}
case scUnknown:
@@ -3056,7 +3058,8 @@ const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
return S;
}
-const SCEV *ScalarEvolution::getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS) {
+const SCEV *ScalarEvolution::getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS,
+ SCEVFlags Flags) {
FoldingSetNodeID ID;
ID.AddInteger(scUDivExpr);
ID.AddPointer(LHS.getOpaqueValue());
@@ -3069,6 +3072,7 @@ const SCEV *ScalarEvolution::getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS) {
S->computeAndSetCanonical(*this);
registerUser(S, {LHS, RHS});
}
+ cast<SCEVUDivExpr>(S)->setExactFlag(Flags);
return S;
}
@@ -3470,11 +3474,16 @@ const SCEV *ScalarEvolution::getURemExpr(SCEVUse LHS, SCEVUse RHS) {
/// Get a canonical unsigned division expression, or something simpler if
/// possible.
-const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
+SCEVUse ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS,
+ SCEVFlagsPair Flags) {
assert(!LHS->getType()->isPointerTy() &&
"SCEVUDivExpr operand can't be pointer!");
assert(LHS->getType() == RHS->getType() &&
"SCEVUDivExpr operand types don't match!");
+ SCEVFlags ExprFlags = Flags.ExprFlags;
+ SCEVFlags UseFlags = Flags.UseFlags;
+ assert(!(ExprFlags & ~SCEV::FlagExact) && "only exact allowed");
+ assert(!(UseFlags & ~SCEV::FlagExact) && "only exact allowed");
if (SCEV *S = findExistingSCEVInCache(scUDivExpr, {LHS, RHS}))
return S;
@@ -3515,7 +3524,7 @@ const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
SCEV::FlagNone)) {
SmallVector<SCEVUse, 4> Operands;
for (const SCEV *Op : AR->operands())
- Operands.push_back(getUDivExpr(Op, RHS));
+ Operands.push_back(getUDivExpr(Op, RHS, Flags));
return getAddRecExpr(Operands, AR->getLoop(), SCEV::FlagNW);
}
/// Get a canonical UDivExpr for a recurrence.
@@ -3545,7 +3554,7 @@ const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
getAddRecExpr(NewStart, Step, AR->getLoop(),
NoWrap ? SCEV::FlagNW : SCEV::FlagNone);
if (LHS != NewLHS)
- return getUDivExpr(NewLHS, RHS);
+ return getUDivExpr(NewLHS, RHS, Flags);
}
}
}
@@ -3555,7 +3564,7 @@ const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
// Find an operand that's safely divisible.
for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
const SCEV *Op = M->getOperand(i);
- const SCEV *Div = getUDivExpr(Op, RHSC);
+ const SCEV *Div = getUDivExpr(Op, RHSC, Flags);
if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) {
SmallVector<SCEVUse, 4> Operands(M->operands());
Operands[i] = Div;
@@ -3572,8 +3581,8 @@ const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
NewOperands.push_back(getConstant(LHSC->getAPInt().udiv(Factor)));
append_range(NewOperands, M->operands().drop_front());
const SCEV *NewMul = getMulExpr(NewOperands);
- return getUDivExpr(NewMul,
- getConstant(RHSC->getAPInt().udiv(Factor)));
+ return getUDivExpr(
+ NewMul, getConstant(RHSC->getAPInt().udiv(Factor)), Flags);
}
}
}
@@ -3589,7 +3598,7 @@ const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
if (Overflow) {
return getConstant(RHSC->getType(), 0, false);
}
- return getUDivExpr(OtherDiv->getLHS(), getConstant(NewRHS));
+ return getUDivExpr(OtherDiv->getLHS(), getConstant(NewRHS), Flags);
}
}
@@ -3599,7 +3608,7 @@ const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
if (A->hasNoUnsignedWrap()) {
SmallVector<SCEVUse, 4> Operands;
for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
- const SCEV *Op = getUDivExpr(A->getOperand(i), RHS);
+ const SCEV *Op = getUDivExpr(A->getOperand(i), RHS, Flags);
if (isa<SCEVUDivExpr>(Op) ||
getMulExpr(Op, RHS) != A->getOperand(i))
break;
@@ -3626,7 +3635,7 @@ const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
*NMinusM == N - *M) {
return getUDivExpr(
getAddExpr(getConstant(N - 1), getMulExpr(getConstant(*M), A)),
- RHS);
+ RHS, Flags);
}
}
@@ -3662,18 +3671,13 @@ const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
const SCEV *NewLHS, *NewRHS;
if (match(LHS, m_scev_c_NUWMul(m_SCEV(NewLHS), m_SCEVVScale())) &&
match(RHS, m_scev_c_NUWMul(m_SCEV(NewRHS), m_SCEVVScale())))
- return getUDivExpr(NewLHS, NewRHS);
+ return getUDivExpr(NewLHS, NewRHS, Flags);
- return getOrCreateUDivExpr(LHS, RHS);
+ return {getOrCreateUDivExpr(LHS, RHS, ExprFlags), UseFlags};
}
-/// Get a canonical unsigned division expression, or something simpler if
-/// possible. There is no representation for an exact udiv in SCEV IR, but we
-/// can attempt to optimize it prior to construction.
const SCEV *ScalarEvolution::getUDivExactExpr(SCEVUse LHS, SCEVUse RHS) {
- // Currently there is no exact specific logic.
-
- return getUDivExpr(LHS, RHS);
+ return getUDivExpr(LHS, RHS, SCEV::FlagExact);
}
/// Get an add recurrence expression for the specified loop. Simplify the
@@ -5197,6 +5201,7 @@ struct BinaryOp {
Value *RHS;
bool IsNSW = false;
bool IsNUW = false;
+ bool IsExact = false;
/// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
/// constant expression.
@@ -5209,6 +5214,8 @@ struct BinaryOp {
IsNSW = OBO->hasNoSignedWrap();
IsNUW = OBO->hasNoUnsignedWrap();
}
+ if (auto *PEO = dyn_cast<PossiblyExactOperator>(Op))
+ IsExact = PEO->isExact();
}
explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
@@ -7938,7 +7945,8 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
case Instruction::UDiv:
LHS = getSCEV(BO->LHS);
RHS = getSCEV(BO->RHS);
- return getUDivExpr(LHS, RHS);
+ return getUDivExpr(LHS, RHS,
+ BO->IsExact ? SCEV::FlagExact : SCEV::FlagNone);
case Instruction::URem:
LHS = getSCEV(BO->LHS);
RHS = getSCEV(BO->RHS);
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index d8ad13994ca2a..9e0da7f41d38c 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -302,9 +302,8 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode, Value *LHS,
if (I->hasNoUnsignedWrap() != any(Flags & SCEV::FlagNUW))
return true;
}
- // Conservatively, do not use any instruction which has any of exact
- // flags installed.
- if (isa<PossiblyExactOperator>(I) && I->isExact())
+ if (isa<PossiblyExactOperator>(I) &&
+ I->isExact() != any(Flags & SCEV::FlagExact))
return true;
return false;
};
@@ -335,6 +334,7 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode, Value *LHS,
Builder.SetCurrentDebugLocation(Loc);
bool IsNUW = any(Flags & SCEV::FlagNUW);
bool IsNSW = any(Flags & SCEV::FlagNSW);
+ bool IsExact = any(Flags & SCEV::FlagExact);
// Don't use folder when expanding post-inc rewrites in LSRMode to preserve
// the rewrites.
if (LSRMode && !PostIncLoops.empty() &&
@@ -346,8 +346,12 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode, Value *LHS,
BO->setHasNoUnsignedWrap();
if (IsNSW)
BO->setHasNoSignedWrap();
+ if (IsExact)
+ BO->setIsExact();
return Builder.Insert(BO);
}
+ if (IsExact)
+ return Builder.CreateExactBinOp(Opcode, LHS, RHS, IsExact);
return Builder.CreateNoWrapBinOp(Opcode, LHS, RHS, IsNUW, IsNSW);
}
@@ -741,7 +745,7 @@ Value *SCEVExpander::visitUDivExpr(SCEVUseT<const SCEVUDivExpr *> S) {
RHS = Builder.CreateIntrinsic(RHS->getType(), Intrinsic::umax,
{RHS, ConstantInt::get(RHS->getType(), 1)});
}
- return InsertBinop(Instruction::UDiv, LHS, RHS, SCEV::FlagNone,
+ return InsertBinop(Instruction::UDiv, LHS, RHS, S.getExactFlag(),
/*IsSafeToHoist*/ SE.isKnownNonZero(S->getRHS()));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index b68040911c142..af3a0c446021a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -217,7 +217,7 @@ const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
});
// A SDiv with non-negative operands is equivalent to an UDiv.
if (match(V, m_SDiv(m_VPValue(LHSVal), m_VPValue(RHSVal)))) {
- return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<SCEVUse> Ops) {
+ return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<SCEVUse> Ops) -> SCEVUse {
if (!SE.isKnownNonNegative(Ops[0]) || !SE.isKnownNonNegative(Ops[1]))
return SE.getCouldNotCompute();
return SE.getUDivExpr(Ops[0], Ops[1]);
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-known-true.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-known-true.ll
index 6ff9f124fa066..caaa9245093e2 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-known-true.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-check-known-true.ll
@@ -15,10 +15,10 @@ define void @test_runtime_check_known_false_after_construction(ptr %start.1, ptr
; CHECK-NEXT: %ptr.iv.2 = phi ptr [ %ptr.iv.2.next, %loop ], [ %start.2.diff, %entry ]
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
-; CHECK-NEXT: (Low: ((-8 * ((2305843009213693951 * (8 + (-1 * (ptrtoaddr ptr %start.1 to i64)) + (ptrtoaddr ptr %end to i64))) /u 8)) + %start.1) High: (8 + %start.1))
+; CHECK-NEXT: (Low: ((-8 * ((2305843009213693951 * (8 + (-1 * (ptrtoaddr ptr %start.1 to i64)) + (ptrtoaddr ptr %end to i64))) /u 8)<exact>) + %start.1) High: (8 + %start.1))
; CHECK-NEXT: Member: {%start.1,+,-8}<%loop>
; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: (-8 + (-8 * ((2305843009213693951 * (8 + (-1 * (ptrtoaddr ptr %start.1 to i64)) + (ptrtoaddr ptr %end to i64))) /u 8)) + (-1 * (ptrtoaddr ptr %start.2 to i64)) + (ptrtoaddr ptr %start.1 to i64) + %start.2) High: ((-1 * (ptrtoaddr ptr %start.2 to i64)) + (ptrtoaddr ptr %start.1 to i64) + %start.2))
+; CHECK-NEXT: (Low: (-8 + (-8 * ((2305843009213693951 * (8 + (-1 * (ptrtoaddr ptr %start.1 to i64)) + (ptrtoaddr ptr %end to i64))) /u 8)<exact>) + (-1 * (ptrtoaddr ptr %start.2 to i64)) + (ptrtoaddr ptr %start.1 to i64) + %start.2) High: ((-1 * (ptrtoaddr ptr %start.2 to i64)) + (ptrtoaddr ptr %start.1 to i64) + %start.2))
; CHECK-NEXT: Member: {(-8 + (-1 * (ptrtoaddr ptr %start.2 to i64)) + (ptrtoaddr ptr %start.1 to i64) + %start.2),+,-8}<%loop>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
diff --git a/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll b/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll
index 506e1f7998ba0..51e3af27ac2e3 100644
--- a/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll
+++ b/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll
@@ -1746,7 +1746,7 @@ define noundef i32 @udiv-basic(i32 %a, i32 %b) {
; CHECK-LABEL: 'udiv-basic'
; CHECK-NEXT: Classifying expressions for: @udiv-basic
; CHECK-NEXT: %res = udiv exact i32 %a, %b
-; CHECK-NEXT: --> (%a /u %b) U: full-set S: full-set
+; CHECK-NEXT: --> (%a /u %b)<exact> U: full-set S: full-set
; CHECK-NEXT: Determining loop execution counts for: @udiv-basic
;
%res = udiv exact i32 %a, %b
diff --git a/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll b/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll
index bb97005e8faf4..703c2e56554d6 100644
--- a/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll
+++ b/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll
@@ -58,13 +58,13 @@ define void @test_well_defined_infinite_st(i32 %N) mustprogress {
; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count.
-; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
; CHECK-NEXT: Loop %for.body: Predicated constant max backedge-taken count is i32 2147483647
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
-; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-2 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-2 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
;
@@ -88,13 +88,13 @@ define void @test_well_defined_infinite_ld(i32 %N) mustprogress {
; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count.
-; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
; CHECK-NEXT: Loop %for.body: Predicated constant max backedge-taken count is i32 2147483647
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
-; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-2 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-2 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
;
@@ -118,13 +118,13 @@ define void @test_no_mustprogress(i32 %N) {
; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count.
-; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
; CHECK-NEXT: Loop %for.body: Predicated constant max backedge-taken count is i32 2147483647
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
-; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-2 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-2 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
;
@@ -214,13 +214,13 @@ define void @test_abnormal_exit(i32 %N) mustprogress {
; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count.
-; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
; CHECK-NEXT: Loop %for.body: Predicated constant max backedge-taken count is i32 2147483647
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
-; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-2 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-2 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
;
@@ -245,7 +245,7 @@ define void @test_other_exit(i32 %N) mustprogress {
; CHECK-NEXT: Loop %for.body: <multiple exits> Unpredictable backedge-taken count.
; CHECK-NEXT: exit count for for.body: i32 9
; CHECK-NEXT: exit count for for.latch: ***COULDNOTCOMPUTE***
-; CHECK-NEXT: predicated exit count for for.latch: ((-2 + %N) /u 2)
+; CHECK-NEXT: predicated exit count for for.latch: ((-2 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
; CHECK-EMPTY:
@@ -253,14 +253,14 @@ define void @test_other_exit(i32 %N) mustprogress {
; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i32 9
; CHECK-NEXT: symbolic max exit count for for.body: i32 9
; CHECK-NEXT: symbolic max exit count for for.latch: ***COULDNOTCOMPUTE***
-; CHECK-NEXT: predicated symbolic max exit count for for.latch: ((-2 + %N) /u 2)
+; CHECK-NEXT: predicated symbolic max exit count for for.latch: ((-2 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
; CHECK-EMPTY:
-; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (9 umin ((-2 + %N) /u 2))
+; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (9 umin ((-2 + %N) /u 2)<exact>)
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
-; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is (9 umin ((-2 + %N) /u 2))
+; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is (9 umin ((-2 + %N) /u 2)<exact>)
; CHECK-NEXT: Predicates:
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i32 %N to i1) to i32) == 0
;
@@ -317,7 +317,7 @@ define void @test_sext(i64 %N) mustprogress {
; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count.
-; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (%N /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (%N /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nssw>
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i64 %N to i1) to i64) == 0
@@ -325,7 +325,7 @@ define void @test_sext(i64 %N) mustprogress {
; CHECK-NEXT: Predicates:
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nssw>
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i64 %N to i1) to i64) == 0
-; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is (%N /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is (%N /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nssw>
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i64 %N to i1) to i64) == 0
@@ -350,7 +350,7 @@ define void @test_zext_of_sext(i64 %N) mustprogress {
; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count.
-; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (%N /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (%N /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nssw>
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nusw>
@@ -360,7 +360,7 @@ define void @test_zext_of_sext(i64 %N) mustprogress {
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nssw>
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nusw>
; CHECK-NEXT: Equal predicate: (zext i1 (trunc i64 %N to i1) to i64) == 0
-; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is (%N /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is (%N /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nssw>
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nusw>
@@ -387,7 +387,7 @@ define void @test_zext_offset(i64 %N) mustprogress {
; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count.
-; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-21 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-21 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nusw>
; CHECK-NEXT: Equal predicate: (zext i1 (true + (trunc i64 %N to i1)) to i64) == 0
@@ -395,7 +395,7 @@ define void @test_zext_offset(i64 %N) mustprogress {
; CHECK-NEXT: Predicates:
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nusw>
; CHECK-NEXT: Equal predicate: (zext i1 (true + (trunc i64 %N to i1)) to i64) == 0
-; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-21 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-21 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nusw>
; CHECK-NEXT: Equal predicate: (zext i1 (true + (trunc i64 %N to i1)) to i64) == 0
@@ -421,7 +421,7 @@ define void @test_sext_offset(i64 %N) mustprogress {
; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count.
; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count.
-; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-21 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-21 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nssw>
; CHECK-NEXT: Equal predicate: (zext i1 (true + (trunc i64 %N to i1)) to i64) == 0
@@ -429,7 +429,7 @@ define void @test_sext_offset(i64 %N) mustprogress {
; CHECK-NEXT: Predicates:
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nssw>
; CHECK-NEXT: Equal predicate: (zext i1 (true + (trunc i64 %N to i1)) to i64) == 0
-; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-21 + %N) /u 2)
+; CHECK-NEXT: Loop %for.body: Predicated symbolic max backedge-taken count is ((-21 + %N) /u 2)<exact>
; CHECK-NEXT: Predicates:
; CHECK-NEXT: {0,+,2}<%for.body> Added Flags: <nssw>
; CHECK-NEXT: Equal predicate: (zext i1 (true + (trunc i64 %N to i1)) to i64) == 0
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-udiv-exact.ll b/llvm/test/Transforms/LoopUnroll/runtime-udiv-exact.ll
index ef7ad8c6b9c32..07a22f3b2cfff 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-udiv-exact.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-udiv-exact.ll
@@ -8,7 +8,7 @@ define void @expand_exact_udiv(i64 %n, ptr %p) {
; CHECK-LABEL: define void @expand_exact_udiv(
; CHECK-SAME: i64 [[N:%.*]], ptr [[P:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 [[N]], 24
+; CHECK-NEXT: [[TMP0:%.*]] = udiv exact i64 [[N]], 24
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i64 [[TMP1]], 1
@@ -68,13 +68,12 @@ define void @reuse_exact_udiv(i64 %n, ptr %p, ptr %q.out) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[Q_PH:%.*]] = udiv exact i64 [[N]], 24
; CHECK-NEXT: store i64 [[Q_PH]], ptr [[Q_OUT]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 [[N]], 24
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[TMP2]], -1
-; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP2]], 1
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[Q_PH]], -1
+; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[Q_PH]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
-; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP2]], [[XTRAITER]]
+; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[Q_PH]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
diff --git a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
index 1fd2eaa5eb72f..4762f2081b1b7 100644
--- a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
+++ b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
@@ -1326,6 +1326,12 @@ TEST_F(ScalarEvolutionsTest, SCEVUseDropsRedundantFlags) {
EXPECT_FALSE(SCEVUse(MaxMixedFlags).isCanonical());
EXPECT_EQ(SCEVUse(MaxMixedFlags).getCanonical(), MaxRedundantFlags);
+ // Redundant exact flag.
+ const SCEV *UDiv = SE.getUDivExpr(X, Y, SCEV::FlagExact);
+ SCEVUse RedundantExactFlags(UDiv, SCEV::FlagExact);
+ EXPECT_FALSE(RedundantExactFlags.hasUseFlags());
+ EXPECT_EQ(RedundantExactFlags, UDiv);
+
// Expressions that cannot carry no-wrap flags themselves must not get use
// flags either, while FlagNone remains fine for them.
const SCEV *ZExt =
@@ -1333,9 +1339,9 @@ TEST_F(ScalarEvolutionsTest, SCEVUseDropsRedundantFlags) {
EXPECT_FALSE(SCEVUse(ZExt, SCEV::FlagNone).hasUseFlags());
#ifndef NDEBUG
EXPECT_DEATH((void)SCEVUse(ZExt, SCEV::FlagNUW),
- "use flags require an expression that can carry no-wrap");
+ "use no-wrap flags require an expression that can carry one");
EXPECT_DEATH((void)SCEVUse(MaxRedundantFlags, SCEV::FlagNSW),
- "use flags require an expression that can carry no-wrap");
+ "use no-wrap flags require an expression that can carry one");
#endif
});
}
>From 4a18dd009c14e45073989a3c07e45f279598dd24 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 30 Sep 2026 09:35:15 +0100
Subject: [PATCH 3/3] [SCEV] Fix nits, strengthen with asserts
---
.../llvm/Analysis/ScalarEvolutionExpressions.h | 12 ++++++++++--
.../lib/Transforms/Utils/ScalarEvolutionExpander.cpp | 4 +++-
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
index 02c1c04190fad..25c5b7b09aea5 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -246,8 +246,11 @@ class SCEVCommutativeExpr : public SCEVNAryExpr {
}
/// Set flags for a non-recurrence without clearing previously set flags.
+ void setFlags(SCEVFlags Flags) {
+ SubclassData |= static_cast<unsigned short>(Flags);
+ }
void setNoWrapFlags(SCEVFlags Flags) {
- SubclassData |= static_cast<unsigned short>(Flags & SCEV::FlagsNoWrapMask);
+ setFlags(Flags & SCEV::FlagsNoWrapMask);
}
};
@@ -310,7 +313,9 @@ class SCEVUDivExpr : public SCEV {
ArrayRef<SCEVUse> operands() const { return Operands; }
+ /// Set the exact flag without clearing previously set flags.
void setExactFlag(SCEVFlags Flags) {
+ assert(!(Flags & ~SCEV::FlagExact) && "Unexpected flags set");
SubclassData |= static_cast<unsigned short>(Flags & SCEV::FlagExact);
}
SCEVFlags getExactFlag() const {
@@ -512,8 +517,11 @@ class SCEVSequentialMinMaxExpr : public SCEVNAryExpr {
}
/// Set flags for a non-recurrence without clearing previously set flags.
+ void setFlags(SCEVFlags Flags) {
+ SubclassData |= static_cast<unsigned short>(Flags);
+ }
void setNoWrapFlags(SCEVFlags Flags) {
- SubclassData |= static_cast<unsigned short>(Flags & SCEV::FlagsNoWrapMask);
+ setFlags(Flags & SCEV::FlagsNoWrapMask);
}
protected:
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 9e0da7f41d38c..9977b1b450457 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -346,8 +346,10 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode, Value *LHS,
BO->setHasNoUnsignedWrap();
if (IsNSW)
BO->setHasNoSignedWrap();
- if (IsExact)
+ if (IsExact) {
+ assert(!IsNUW && !IsNSW && "Unexpected nuw/nsw found");
BO->setIsExact();
+ }
return Builder.Insert(BO);
}
if (IsExact)
More information about the llvm-commits
mailing list