[llvm] [SCEV] Preserve divisibility info when creating UMax/SMax expressions. (PR #160012)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 6 07:43:44 PDT 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/160012
>From 23d922eef778e7d6818fdbb0055977c9558ebea2 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 21 Sep 2025 20:33:41 +0100
Subject: [PATCH 1/2] [SCEV] Preserve divisibility info when creating UMax/SMax
expressions.
Currently we generate (S|U)Max(1, Op) for Op >= 1. This may discard
divisibility info of Op. This patch rewrites such SMax/UMax expressions
to use the lowest common multiplier for all non-constant operands.
---
llvm/lib/Analysis/ScalarEvolution.cpp | 24 +++++++++++++++++--
.../ScalarEvolution/trip-count-minmax.ll | 4 ++--
2 files changed, 24 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 63e1b1462d007..4be503e71daa9 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -15857,12 +15857,17 @@ void ScalarEvolution::LoopGuards::collectFromBlock(
To = SE.getUMaxExpr(FromRewritten, RHS);
if (auto *UMin = dyn_cast<SCEVUMinExpr>(FromRewritten))
EnqueueOperands(UMin);
+ if (RHS->isOne())
+ ExprsToRewrite.push_back(From);
break;
case CmpInst::ICMP_SGT:
case CmpInst::ICMP_SGE:
To = SE.getSMaxExpr(FromRewritten, RHS);
- if (auto *SMin = dyn_cast<SCEVSMinExpr>(FromRewritten))
+ if (auto *SMin = dyn_cast<SCEVSMinExpr>(FromRewritten)) {
EnqueueOperands(SMin);
+ }
+ if (RHS->isOne())
+ ExprsToRewrite.push_back(From);
break;
case CmpInst::ICMP_EQ:
if (isa<SCEVConstant>(RHS))
@@ -15993,7 +15998,22 @@ void ScalarEvolution::LoopGuards::collectFromBlock(
for (const SCEV *Expr : ExprsToRewrite) {
const SCEV *RewriteTo = Guards.RewriteMap[Expr];
Guards.RewriteMap.erase(Expr);
- Guards.RewriteMap.insert({Expr, Guards.rewrite(RewriteTo)});
+ const SCEV *Rewritten = Guards.rewrite(RewriteTo);
+
+ // Try to strengthen divisibility of SMax/UMax expressions coming from >=
+ // 1 conditions.
+ if (auto *SMax = dyn_cast<SCEVSMaxExpr>(Rewritten)) {
+ unsigned MinTrailingZeros = SE.getMinTrailingZeros(SMax->getOperand(1));
+ for (const SCEV *Op : drop_begin(SMax->operands(), 2))
+ MinTrailingZeros =
+ std::min(MinTrailingZeros, SE.getMinTrailingZeros(Op));
+ if (MinTrailingZeros != 0)
+ Rewritten = SE.getSMaxExpr(
+ SE.getConstant(APInt(SMax->getType()->getScalarSizeInBits(), 1)
+ .shl(MinTrailingZeros)),
+ SMax);
+ }
+ Guards.RewriteMap.insert({Expr, Rewritten});
}
}
}
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll
index 8d091a00ed4b9..d38010403dad7 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll
@@ -61,7 +61,7 @@ define void @umin(i32 noundef %a, i32 noundef %b) {
; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a) umin (4 * %b)))
; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646
; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a) umin (4 * %b)))
-; CHECK-NEXT: Loop %for.body: Trip multiple is 1
+; CHECK-NEXT: Loop %for.body: Trip multiple is 2
;
; void umin(unsigned a, unsigned b) {
; a *= 2;
@@ -157,7 +157,7 @@ define void @smin(i32 noundef %a, i32 noundef %b) {
; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a)<nsw> smin (4 * %b)<nsw>))
; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646
; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a)<nsw> smin (4 * %b)<nsw>))
-; CHECK-NEXT: Loop %for.body: Trip multiple is 1
+; CHECK-NEXT: Loop %for.body: Trip multiple is 2
;
; void smin(signed a, signed b) {
; a *= 2;
>From 8e50ec5f9b82549730f8bfb786e5bd37e31fbc53 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 22 Sep 2025 19:53:24 +0100
Subject: [PATCH 2/2] !fixup also rewrite SCEVUMaxExpr, use
getConstantMultiple.
---
llvm/lib/Analysis/ScalarEvolution.cpp | 20 ++++----
.../LoopVectorize/single_early_exit.ll | 47 ++++++++++++++++---
2 files changed, 50 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 4be503e71daa9..40dc1b453d28c 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -16002,16 +16002,16 @@ void ScalarEvolution::LoopGuards::collectFromBlock(
// Try to strengthen divisibility of SMax/UMax expressions coming from >=
// 1 conditions.
- if (auto *SMax = dyn_cast<SCEVSMaxExpr>(Rewritten)) {
- unsigned MinTrailingZeros = SE.getMinTrailingZeros(SMax->getOperand(1));
- for (const SCEV *Op : drop_begin(SMax->operands(), 2))
- MinTrailingZeros =
- std::min(MinTrailingZeros, SE.getMinTrailingZeros(Op));
- if (MinTrailingZeros != 0)
- Rewritten = SE.getSMaxExpr(
- SE.getConstant(APInt(SMax->getType()->getScalarSizeInBits(), 1)
- .shl(MinTrailingZeros)),
- SMax);
+ auto *Max = dyn_cast<SCEVMinMaxExpr>(Rewritten);
+ if (Max && isa<SCEVSMaxExpr, SCEVUMaxExpr>(Rewritten) &&
+ Rewritten->getType()->isIntegerTy() && Max->getOperand(0)->isOne()) {
+ APInt CommonMultiple = SE.getConstantMultiple(Max->getOperand(1));
+ for (const SCEV *Op : drop_begin(Max->operands(), 2)) {
+ CommonMultiple = APIntOps::GreatestCommonDivisor(
+ CommonMultiple, SE.getConstantMultiple(Op));
+ }
+ SmallVector<const SCEV *> Ops = {SE.getConstant(CommonMultiple), Max};
+ Rewritten = SE.getMinMaxExpr(Max->getSCEVType(), Ops);
}
Guards.RewriteMap.insert({Expr, Rewritten});
}
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
index 3500c5c9d81cd..4fd8d17073de4 100644
--- a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
@@ -546,19 +546,50 @@ define i64 @loop_guards_needed_to_prove_deref_multiple(i32 %x, i1 %c, ptr derefe
; CHECK-NEXT: call void @llvm.assume(i1 [[PRE_2]])
; CHECK-NEXT: [[N:%.*]] = add i32 [[SEL]], -1
; CHECK-NEXT: [[N_EXT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[SEL]], -2
+; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[IV_NEXT:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_HEADER]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[IV_NEXT]]
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: middle.split:
+; CHECK-NEXT: br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[LOOP_LATCH:%.*]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[IV_NEXT]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: vector.early.exit:
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT: br label [[EXIT_LOOPEXIT]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT]], [[LOOP_LATCH]] ], [ 0, [[PH]] ]
+; CHECK-NEXT: br label [[LOOP_HEADER1:%.*]]
; CHECK: loop.header:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[PH]] ]
-; CHECK-NEXT: [[GEP_SRC_I:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT1:%.*]], [[LOOP_LATCH1:%.*]] ], [ [[IV]], [[SCALAR_PH]] ]
+; CHECK-NEXT: [[GEP_SRC_I:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP_SRC_I]], align 1
; CHECK-NEXT: [[C_1:%.*]] = icmp eq i8 [[L]], 0
-; CHECK-NEXT: br i1 [[C_1]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_LATCH]]
+; CHECK-NEXT: br i1 [[C_1]], label [[EXIT_LOOPEXIT]], label [[LOOP_LATCH1]]
; CHECK: loop.latch:
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N_EXT]]
-; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER]]
+; CHECK-NEXT: [[IV_NEXT1]] = add i64 [[IV1]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N_EXT]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER1]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: exit.loopexit:
-; CHECK-NEXT: [[RES_PH:%.*]] = phi i64 [ [[IV]], [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RES_PH:%.*]] = phi i64 [ [[IV1]], [[LOOP_HEADER1]] ], [ 0, [[LOOP_LATCH1]] ], [ 0, [[LOOP_LATCH]] ], [ [[TMP10]], [[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT: br label [[EXIT]]
; CHECK: exit:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ -1, [[ENTRY:%.*]] ], [ -2, [[THEN]] ], [ [[RES_PH]], [[EXIT_LOOPEXIT]] ]
@@ -609,4 +640,6 @@ exit:
; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META2]], [[META1]]}
;.
More information about the llvm-commits
mailing list