[llvm] da4e5cc - [SCEV] Canonicalise round-up idiom when some bits known (#197126)
via llvm-commits
llvm-commits at lists.llvm.org
Thu May 28 06:29:15 PDT 2026
Author: Oliver Stannard
Date: 2026-05-28T14:29:09+01:00
New Revision: da4e5cc928c8f8000643e48bc2790b51927eeac6
URL: https://github.com/llvm/llvm-project/commit/da4e5cc928c8f8000643e48bc2790b51927eeac6
DIFF: https://github.com/llvm/llvm-project/commit/da4e5cc928c8f8000643e48bc2790b51927eeac6.diff
LOG: [SCEV] Canonicalise round-up idiom when some bits known (#197126)
Since #174380, instcombine can clear some set bits in the added constant
in expressions like this, when A has some known-clear low order bits.
(A + 15) & ~15
This transformation is valid, but can make it harder for later passes to
recognise this idiom for rounding up to a power of 2. This is causing
the ARM MVE tail predication pass to fail on loops with a trip count
which is a multiple of a small power of 2.
The fix is to reverse the transformation when building SCEV expressions,
canonicalising to always use the largest valid value for the added
constant.
Alive proofs:
https://alive2.llvm.org/ce/z/hhndoW
https://alive2.llvm.org/ce/z/_JYVat
Added:
llvm/test/Analysis/ScalarEvolution/rounding-demanded-bits.ll
llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-known-bits.ll
Modified:
llvm/lib/Analysis/ScalarEvolution.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 855ab3bb5d621..ca92aa2745b6e 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -3718,6 +3718,26 @@ const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
}
}
+ // ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
+ // This is an idiom for rounding A up to the next multiple of N, where A
+ // is aready known to be a multiple of M. In this case, instcombine can
+ // see that some low bits of the added constant are unused, so can clear
+ // them, but we want to canonicalise to set the low bits. This makes the
+ // pattern easier to match, without needing to check for known bits in
+ // A*M.
+ const APInt &N = RHSC->getAPInt();
+ const APInt *NMinusM, *M;
+ const SCEV *A;
+ if (match(LHS, m_scev_Add(m_scev_APInt(NMinusM),
+ m_scev_Mul(m_scev_APInt(M), m_SCEV(A))))) {
+ if (N.isPowerOf2() && M->isPowerOf2() && M->ult(N) &&
+ *NMinusM == N - *M) {
+ return getUDivExpr(
+ getAddExpr(getConstant(N - 1), getMulExpr(getConstant(*M), A)),
+ RHS);
+ }
+ }
+
// Fold if both operands are constant.
if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS))
return getConstant(LHSC->getAPInt().udiv(RHSC->getAPInt()));
diff --git a/llvm/test/Analysis/ScalarEvolution/rounding-demanded-bits.ll b/llvm/test/Analysis/ScalarEvolution/rounding-demanded-bits.ll
new file mode 100644
index 0000000000000..116b0278eeebf
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/rounding-demanded-bits.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s "-passes=print<scalar-evolution>" -disable-output 2>&1 | FileCheck %s
+
+; Check that, when constructing a SCEV for rounding a value up to the nearest
+; multiple of a constant, we always use that largest possible value for the
+; added constant, even if instcombine has cleared some bits in the IR because
+; the input is already known to be a multiple of some smaller value.
+
+define i32 @mul_2_round_to_16_a(i32 %val) {
+; CHECK-LABEL: 'mul_2_round_to_16_a'
+; CHECK-NEXT: Classifying expressions for: @mul_2_round_to_16_a
+; CHECK-NEXT: %mul = mul i32 %val, 2
+; CHECK-NEXT: --> (2 * %val) U: [0,-1) S: [-2147483648,2147483647)
+; CHECK-NEXT: %add = add i32 %mul, 15
+; CHECK-NEXT: --> (15 + (2 * %val)) U: [15,14) S: [-2147483633,-2147483634)
+; CHECK-NEXT: %round = and i32 %add, -16
+; CHECK-NEXT: --> (16 * ((15 + (2 * %val)) /u 16))<nuw> U: [0,-15) S: [-2147483648,2147483633)
+; CHECK-NEXT: Determining loop execution counts for: @mul_2_round_to_16_a
+;
+ %mul = mul i32 %val, 2
+ %add = add i32 %mul, 15
+ %round = and i32 %add, -16
+ ret i32 %round
+}
+
+define i32 @mul_2_round_to_16_b(i32 %val) {
+; CHECK-LABEL: 'mul_2_round_to_16_b'
+; CHECK-NEXT: Classifying expressions for: @mul_2_round_to_16_b
+; CHECK-NEXT: %mul = mul i32 %val, 2
+; CHECK-NEXT: --> (2 * %val) U: [0,-1) S: [-2147483648,2147483647)
+; CHECK-NEXT: %add = add i32 %mul, 14
+; CHECK-NEXT: --> (14 + (2 * %val)) U: [0,-1) S: [-2147483648,2147483647)
+; CHECK-NEXT: %round = and i32 %add, -16
+; CHECK-NEXT: --> (16 * ((15 + (2 * %val)) /u 16))<nuw> U: [0,-15) S: [-2147483648,2147483633)
+; CHECK-NEXT: Determining loop execution counts for: @mul_2_round_to_16_b
+;
+ %mul = mul i32 %val, 2
+ %add = add i32 %mul, 14
+ %round = and i32 %add, -16
+ ret i32 %round
+}
+
+define i32 @mul_2_round_to_16_c(i32 %val) {
+; CHECK-LABEL: 'mul_2_round_to_16_c'
+; CHECK-NEXT: Classifying expressions for: @mul_2_round_to_16_c
+; CHECK-NEXT: %mul = mul i32 %val, 2
+; CHECK-NEXT: --> (2 * %val) U: [0,-1) S: [-2147483648,2147483647)
+; CHECK-NEXT: %add = add i32 %mul, 14
+; CHECK-NEXT: --> (14 + (2 * %val)) U: [0,-1) S: [-2147483648,2147483647)
+; CHECK-NEXT: %div = udiv i32 %add, 16
+; CHECK-NEXT: --> ((15 + (2 * %val)) /u 16) U: [0,268435456) S: [0,268435456)
+; CHECK-NEXT: %round = mul i32 %div, 16
+; CHECK-NEXT: --> (16 * ((15 + (2 * %val)) /u 16))<nuw> U: [0,-15) S: [-2147483648,2147483633)
+; CHECK-NEXT: Determining loop execution counts for: @mul_2_round_to_16_c
+;
+ %mul = mul i32 %val, 2
+ %add = add i32 %mul, 14
+ %div = udiv i32 %add, 16
+ %round = mul i32 %div, 16
+ ret i32 %round
+}
+
+define i32 @mul_4_round_to_16(i32 %val) {
+; CHECK-LABEL: 'mul_4_round_to_16'
+; CHECK-NEXT: Classifying expressions for: @mul_4_round_to_16
+; CHECK-NEXT: %mul = mul i32 %val, 4
+; CHECK-NEXT: --> (4 * %val) U: [0,-3) S: [-2147483648,2147483645)
+; CHECK-NEXT: %add = add i32 %mul, 12
+; CHECK-NEXT: --> (12 + (4 * %val)) U: [0,-3) S: [-2147483648,2147483645)
+; CHECK-NEXT: %round = and i32 %add, -16
+; CHECK-NEXT: --> (16 * ((15 + (4 * %val)) /u 16))<nuw> U: [0,-15) S: [-2147483648,2147483633)
+; CHECK-NEXT: Determining loop execution counts for: @mul_4_round_to_16
+;
+ %mul = mul i32 %val, 4
+ %add = add i32 %mul, 12
+ %round = and i32 %add, -16
+ ret i32 %round
+}
+
+; N is not a power of 2, cannot do transformation
+define i32 @invalid1(i32 %val) {
+; CHECK-LABEL: 'invalid1'
+; CHECK-NEXT: Classifying expressions for: @invalid1
+; CHECK-NEXT: %mul = mul i32 %val, 3
+; CHECK-NEXT: --> (3 * %val) U: full-set S: full-set
+; CHECK-NEXT: %add = add i32 %mul, 7
+; CHECK-NEXT: --> (7 + (3 * %val)) U: full-set S: full-set
+; CHECK-NEXT: %div = udiv i32 %add, 9
+; CHECK-NEXT: --> ((7 + (3 * %val)) /u 9) U: [0,477218589) S: [0,477218589)
+; CHECK-NEXT: %round = mul i32 %div, 9
+; CHECK-NEXT: --> (9 * ((7 + (3 * %val)) /u 9))<nuw> U: [0,-3) S: [0,-3)
+; CHECK-NEXT: Determining loop execution counts for: @invalid1
+;
+ %mul = mul i32 %val, 3
+ %add = add i32 %mul, 7
+ %div = udiv i32 %add, 9
+ %round = mul i32 %div, 9
+ ret i32 %round
+}
+
+; M is greater then N, cannot do transformation
+define i32 @invalid2(i32 %val) {
+; CHECK-LABEL: 'invalid2'
+; CHECK-NEXT: Classifying expressions for: @invalid2
+; CHECK-NEXT: %mul = mul i32 %val, 4
+; CHECK-NEXT: --> (4 * %val) U: [0,-3) S: [-2147483648,2147483645)
+; CHECK-NEXT: %add = add i32 %mul, 1
+; CHECK-NEXT: --> (1 + (4 * %val))<nuw><nsw> U: [1,-2) S: [-2147483647,2147483646)
+; CHECK-NEXT: %round = and i32 %add, -2
+; CHECK-NEXT: --> (2 * ((1 + (4 * %val))<nuw><nsw> /u 2))<nuw> U: [0,-3) S: [-2147483648,2147483647)
+; CHECK-NEXT: Determining loop execution counts for: @invalid2
+;
+ %mul = mul i32 %val, 4
+ %add = add i32 %mul, 1
+ %round = and i32 %add, -2
+ ret i32 %round
+}
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-known-bits.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-known-bits.ll
new file mode 100644
index 0000000000000..07abf1c2456e3
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-known-bits.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve %s -S -o - | FileCheck %s
+
+; Check that tail predication still triggers correctly when the back-edge taken
+; count is known to be a multiple of 2.
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read)
+define dso_local i32 @example(ptr noundef readonly captures(none) %ptr, i32 noundef %n) {
+; CHECK-LABEL: define dso_local i32 @example(
+; CHECK-SAME: ptr noundef readonly captures(none) [[PTR:%.*]], i32 noundef [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP3:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP3]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[MUL:%.*]] = shl i32 [[N]], 1
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[MUL]], 14
+; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N_VEC]], -16
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP2]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[LSR_IV:%.*]] = phi ptr [ [[SCEVGEP:%.*]], %[[VECTOR_BODY]] ], [ [[PTR]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ [[TMP3]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi i32 [ [[MUL]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = call <16 x i1> @llvm.arm.mve.vctp8(i32 [[TMP5]])
+; CHECK-NEXT: [[TMP7]] = sub i32 [[TMP5]], 16
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[LSR_IV]], <16 x i1> [[TMP6]], <16 x i8> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[TMP9:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP9]])
+; CHECK-NEXT: [[TMP11]] = add i32 [[VEC_PHI]], [[TMP10]]
+; CHECK-NEXT: [[SCEVGEP]] = getelementptr i8, ptr [[LSR_IV]], i32 16
+; CHECK-NEXT: [[TMP12]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP4]], i32 1)
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i32 [[TMP12]], 0
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_BODY]], label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP11]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
+;
+entry:
+ %cmp3 = icmp sgt i32 %n, 0
+ br i1 %cmp3, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ %mul = shl nuw i32 %n, 1
+ %n.rnd.up = add i32 %mul, 14
+ %n.vec = and i32 %n.rnd.up, -16
+ %0 = add i32 %n.vec, -16
+ %1 = lshr i32 %0, 4
+ %2 = add nuw nsw i32 %1, 1
+ %3 = call i32 @llvm.start.loop.iterations.i32(i32 %2)
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %for.body.preheader
+ %lsr.iv = phi ptr [ %scevgep, %vector.body ], [ %ptr, %for.body.preheader ]
+ %index = phi i32 [ 0, %for.body.preheader ], [ %index.next, %vector.body ]
+ %vec.phi = phi i32 [ 0, %for.body.preheader ], [ %8, %vector.body ]
+ %4 = phi i32 [ %3, %for.body.preheader ], [ %9, %vector.body ]
+ %active.lane.mask = tail call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %mul)
+ %wide.masked.load = tail call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 %lsr.iv, <16 x i1> %active.lane.mask, <16 x i8> poison)
+ %5 = zext <16 x i8> %wide.masked.load to <16 x i32>
+ %6 = select <16 x i1> %active.lane.mask, <16 x i32> %5, <16 x i32> zeroinitializer
+ %7 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %6)
+ %8 = add i32 %vec.phi, %7
+ %index.next = add i32 %index, 16
+ %scevgep = getelementptr i8, ptr %lsr.iv, i32 16
+ %9 = call i32 @llvm.loop.decrement.reg.i32(i32 %4, i32 1)
+ %10 = icmp ne i32 %9, 0
+ br i1 %10, label %vector.body, label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %vector.body, %entry
+ %sum.0.lcssa = phi i32 [ 0, %entry ], [ %8, %vector.body ]
+ ret i32 %sum.0.lcssa
+}
More information about the llvm-commits
mailing list