[llvm] [SE] Fold zext(C + X)<nsw> to sext(C) + zext(X) when X >=u |C| (PR #205926)
Aleksandr Popov via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 02:33:55 PDT 2026
https://github.com/aleks-tmb updated https://github.com/llvm/llvm-project/pull/205926
>From ab860dd41957a19e89718aa0093870682c1a91d5 Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Fri, 26 Jun 2026 09:22:39 +0000
Subject: [PATCH 1/2] Precommit single_early_exit_zext_trip_count.ll test
---
.../single_early_exit_zext_trip_count.ll | 79 +++++++++++++++++++
1 file changed, 79 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/single_early_exit_zext_trip_count.ll
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit_zext_trip_count.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit_zext_trip_count.ll
new file mode 100644
index 0000000000000..25b2512b15d22
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit_zext_trip_count.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 | FileCheck %s
+
+; Trip count is zext(add nsw %len, -1) where %len is known non-negative via !range.
+; TODO: SCEV must fold the zext through the add to compute the backedge-taken
+; count, enabling vectorization.
+define void @early_exit_zext_trip_count(ptr %p) {
+; CHECK-LABEL: define void @early_exit_zext_trip_count(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[BASE:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
+; CHECK-NEXT: [[LEN:%.*]] = load atomic i32, ptr [[BASE]] unordered, align 4, !range [[RNG0:![0-9]+]]
+; CHECK-NEXT: [[LEN_PLUS_8:%.*]] = add nuw nsw i32 [[LEN]], 8
+; CHECK-NEXT: [[LEN_PLUS_8_WIDE:%.*]] = zext i32 [[LEN_PLUS_8]] to i64
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 [[LEN_PLUS_8_WIDE]]) ]
+; CHECK-NEXT: [[LEN_IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0
+; CHECK-NEXT: br i1 [[LEN_IS_ZERO]], label [[RET:%.*]], label [[PREHEADER:%.*]]
+; CHECK: preheader:
+; CHECK-NEXT: [[EXIT_32:%.*]] = add nsw i32 [[LEN]], -1
+; CHECK-NEXT: [[EXIT:%.*]] = zext i32 [[EXIT_32]] to i64
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ], [ 0, [[PREHEADER]] ]
+; CHECK-NEXT: [[ELEM_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 [[IV]]
+; CHECK-NEXT: [[ELEM:%.*]] = load i8, ptr [[ELEM_PTR]], align 1
+; CHECK-NEXT: [[IS_TARGET:%.*]] = icmp eq i8 [[ELEM]], 0
+; CHECK-NEXT: br i1 [[IS_TARGET]], label [[DEOPT:%.*]], label [[LATCH]]
+; CHECK: latch:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp ult i64 [[IV]], [[EXIT]]
+; CHECK-NEXT: br i1 [[LOOP_COND]], label [[LOOP]], label [[RET_LOOPEXIT:%.*]]
+; CHECK: ret.loopexit:
+; CHECK-NEXT: br label [[RET]]
+; CHECK: ret:
+; CHECK-NEXT: ret void
+; CHECK: deopt:
+; CHECK-NEXT: call void @deopt()
+; CHECK-NEXT: unreachable
+;
+entry:
+ %base = getelementptr inbounds nuw i8, ptr %p, i64 8
+ %len = load atomic i32, ptr %base unordered, align 4, !range !0
+ %len.plus.8 = add nuw nsw i32 %len, 8
+ %len.plus.8.wide = zext i32 %len.plus.8 to i64
+ call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %p, i64 %len.plus.8.wide) ]
+ %len.is.zero = icmp eq i32 %len, 0
+ br i1 %len.is.zero, label %ret, label %preheader
+
+preheader: ; preds = %entry
+ %exit.32 = add nsw i32 %len, -1
+ %exit = zext i32 %exit.32 to i64
+ br label %loop
+
+loop: ; preds = %latch, %preheader
+ %iv = phi i64 [ %iv.next, %latch ], [ 0, %preheader ]
+ %elem.ptr = getelementptr inbounds nuw i8, ptr %base, i64 %iv
+ %elem = load i8, ptr %elem.ptr, align 1
+ %is.target = icmp eq i8 %elem, 0
+ br i1 %is.target, label %deopt, label %latch
+
+latch: ; preds = %loop
+ %iv.next = add nuw nsw i64 %iv, 1
+ %loop.cond = icmp ult i64 %iv, %exit
+ br i1 %loop.cond, label %loop, label %ret
+
+ret: ; preds = %latch, %entry
+ ret void
+
+deopt: ; preds = %loop
+ call void @deopt()
+ unreachable
+}
+
+declare void @deopt()
+
+!0 = !{i32 0, i32 1000}
+;.
+; CHECK: [[RNG0]] = !{i32 0, i32 1000}
+;.
>From 6d8fd1f698fc3682517249508dc6a58f2389fcc9 Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Thu, 25 Jun 2026 20:56:29 +0000
Subject: [PATCH 2/2] [SCEV] Fold zext(C + X)<nsw> to sext(C) + zext(X) when X
>=u |C|
When a zero-extended add has a negative constant and nsw, the zext
can be split: the constant is sign-extended while the non-constant
part is zero-extended. This is valid because X >=u |C| guarantees
no unsigned underflow, so zext(X - K) == zext(X) - K.
This is a generalization of the existing smax fold directly above,
which handles the special case where X is smax(C2, A). Unlike that
fold, X is zero-extended (not sign-extended) because it may have
the source sign bit set.
---
llvm/lib/Analysis/ScalarEvolution.cpp | 18 ++++++++
.../max-backedge-taken-count-guard-info.ll | 2 +-
.../ScalarEvolution/zext-add-nsw-fold.ll | 8 ++--
.../LoopIdiom/X86/memset-size-compute.ll | 8 ++--
.../single_early_exit_zext_trip_count.ll | 46 +++++++++++++++----
5 files changed, 65 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index c0cdce982e623..6350fb69c9bb0 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -1924,6 +1924,24 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
SCEV::FlagNSW, Depth + 1);
}
+ // zext(C + X)<nsw> -> (sext(C) + zext(X))<nsw>
+ // for negative C, if X >=u abs(C) (no unsigned underflow).
+ //
+ // Let K = abs(C). Since X >=u K, X + C == X - K does not
+ // unsigned-underflow in the source type, so
+ // zext(X - K) == zext(X) - K == zext(X) + sext(C).
+ //
+ // Unlike the smax case above, X may be unsigned-large (sign bit set),
+ // so it must be zero-extended; sign-extending would change its value.
+ if (SA->hasNoSignedWrap() && SA->getNumOperands() == 2 &&
+ match(SA, m_scev_Add(m_scev_APInt(C), m_SCEV())) && C->isNegative() &&
+ !C->isMinSignedValue() &&
+ getUnsignedRangeMin(SA->getOperand(1)).uge(C->abs())) {
+ return getAddExpr(getSignExtendExpr(SA->getOperand(0), Ty, Depth + 1),
+ getZeroExtendExpr(SA->getOperand(1), Ty, Depth + 1),
+ SCEV::FlagNSW, Depth + 1);
+ }
+
// zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
// if D + (C - D + x + y + ...) could be proven to not unsigned wrap
// where D maximizes the number of trailing zeros of (C - D + x + y + ...)
diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
index 436f0e55840d9..449dbab51f325 100644
--- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
+++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
@@ -1231,7 +1231,7 @@ define void @optimized_range_check_unsigned3(ptr %pred, i1 %c) {
; CHECK-NEXT: %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK-NEXT: --> {0,+,1}<nuw><nsw><%loop> U: [0,3) S: [0,3) Exits: (-1 + %N)<nsw> LoopDispositions: { %loop: Computable }
; CHECK-NEXT: %gep = getelementptr inbounds i16, ptr %pred, i32 %iv
-; CHECK-NEXT: --> {%pred,+,2}<nuw><%loop> U: full-set S: full-set Exits: ((zext i32 (-2 + (2 * %N)<nuw><nsw>)<nsw> to i64) + %pred) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT: --> {%pred,+,2}<nuw><%loop> U: full-set S: full-set Exits: (-2 + (2 * (zext i32 %N to i64))<nuw><nsw> + %pred) LoopDispositions: { %loop: Computable }
; CHECK-NEXT: %iv.next = add nuw nsw i32 %iv, 1
; CHECK-NEXT: --> {1,+,1}<nuw><nsw><%loop> U: [1,4) S: [1,4) Exits: %N LoopDispositions: { %loop: Computable }
; CHECK-NEXT: Determining loop execution counts for: @optimized_range_check_unsigned3
diff --git a/llvm/test/Analysis/ScalarEvolution/zext-add-nsw-fold.ll b/llvm/test/Analysis/ScalarEvolution/zext-add-nsw-fold.ll
index 06e1f48a5f2c4..6a9aacda7254c 100644
--- a/llvm/test/Analysis/ScalarEvolution/zext-add-nsw-fold.ll
+++ b/llvm/test/Analysis/ScalarEvolution/zext-add-nsw-fold.ll
@@ -84,7 +84,7 @@ define i64 @mul_umax_fold(i8 %x) {
; CHECK-NEXT: %add = add nsw i32 -12, %mul
; CHECK-NEXT: --> (-12 + (4 * (3 umax (zext i8 %x to i32)))<nuw><nsw>)<nsw> U: [0,1009) S: [0,1009)
; CHECK-NEXT: %ext = zext i32 %add to i64
-; CHECK-NEXT: --> (zext i32 (-12 + (4 * (3 umax (zext i8 %x to i32)))<nuw><nsw>)<nsw> to i64) U: [0,1009) S: [0,1009)
+; CHECK-NEXT: --> (-12 + (4 * (3 umax (zext i8 %x to i64)))<nuw><nsw>)<nsw> U: [0,1009) S: [0,1009)
; CHECK-NEXT: Determining loop execution counts for: @mul_umax_fold
;
%zx = zext i8 %x to i32
@@ -107,7 +107,7 @@ define i64 @mul_umax_fold_round_up(i8 %x) {
; CHECK-NEXT: %add = add nsw i32 -10, %mul
; CHECK-NEXT: --> (-10 + (4 * (3 umax (zext i8 %x to i32)))<nuw><nsw>)<nsw> U: [2,1011) S: [2,1011)
; CHECK-NEXT: %ext = zext i32 %add to i64
-; CHECK-NEXT: --> (2 + (zext i32 (-12 + (4 * (3 umax (zext i8 %x to i32)))<nuw><nsw>)<nsw> to i64))<nuw><nsw> U: [2,1011) S: [2,1011)
+; CHECK-NEXT: --> (-10 + (4 * (3 umax (zext i8 %x to i64)))<nuw><nsw>)<nsw> U: [2,1011) S: [2,1011)
; CHECK-NEXT: Determining loop execution counts for: @mul_umax_fold_round_up
;
%zx = zext i8 %x to i32
@@ -153,7 +153,7 @@ define i64 @mul_smax_fold(i8 %x) {
; CHECK-NEXT: %add = add nsw i32 -8, %mul
; CHECK-NEXT: --> (-8 + (4 * (3 smax (zext i8 %x to i32)))<nuw><nsw>)<nsw> U: [4,1013) S: [4,1013)
; CHECK-NEXT: %ext = zext i32 %add to i64
-; CHECK-NEXT: --> (zext i32 (-8 + (4 * (3 smax (zext i8 %x to i32)))<nuw><nsw>)<nsw> to i64) U: [4,1013) S: [4,1013)
+; CHECK-NEXT: --> (-8 + (4 * (zext i32 (3 smax (zext i8 %x to i32)) to i64))<nuw><nsw>)<nsw> U: [4,1013) S: [4,1013)
; CHECK-NEXT: Determining loop execution counts for: @mul_smax_fold
;
%zx = zext i8 %x to i32
@@ -219,7 +219,7 @@ define i64 @umax_zext_fold(i8 %x) {
; CHECK-NEXT: %add = add nsw i32 -5, %umax
; CHECK-NEXT: --> (-5 + (5 umax (zext i8 %x to i32)))<nsw> U: [0,251) S: [0,251)
; CHECK-NEXT: %ext = zext i32 %add to i64
-; CHECK-NEXT: --> (zext i32 (-5 + (5 umax (zext i8 %x to i32)))<nsw> to i64) U: [0,251) S: [0,251)
+; CHECK-NEXT: --> (-5 + (5 umax (zext i8 %x to i64)))<nsw> U: [0,251) S: [0,251)
; CHECK-NEXT: Determining loop execution counts for: @umax_zext_fold
;
%zx = zext i8 %x to i32
diff --git a/llvm/test/Transforms/LoopIdiom/X86/memset-size-compute.ll b/llvm/test/Transforms/LoopIdiom/X86/memset-size-compute.ll
index 0123f15334281..b58e3c7d275bb 100644
--- a/llvm/test/Transforms/LoopIdiom/X86/memset-size-compute.ll
+++ b/llvm/test/Transforms/LoopIdiom/X86/memset-size-compute.ll
@@ -15,10 +15,10 @@ define void @test(ptr %ptr) {
; CHECK: for.body.preheader:
; CHECK-NEXT: [[LIM_0:%.*]] = phi i32 [ 65, [[ENTRY:%.*]] ], [ 1, [[DEAD:%.*]] ]
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 8
-; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[LIM_0]], i32 2)
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i32 [[UMAX]], 3
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[TMP2]], -8
-; CHECK-NEXT: [[TMP1:%.*]] = zext nneg i32 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[LIM_0]] to i64
+; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 2)
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[UMAX]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP2]], -8
; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 [[SCEVGEP]], i8 0, i64 [[TMP1]], i1 false)
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit_zext_trip_count.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit_zext_trip_count.ll
index 25b2512b15d22..2d5eff498de9a 100644
--- a/llvm/test/Transforms/LoopVectorize/single_early_exit_zext_trip_count.ll
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit_zext_trip_count.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 | FileCheck %s
-; Trip count is zext(add nsw %len, -1) where %len is known non-negative via !range.
-; TODO: SCEV must fold the zext through the add to compute the backedge-taken
+; Trip count is zext(add nsw %len, -1) where %len is known non-negative via !range.
+; SCEV must fold the zext through the add to compute the backedge-taken
; count, enabling vectorization.
define void @early_exit_zext_trip_count(ptr %p) {
; CHECK-LABEL: define void @early_exit_zext_trip_count(
@@ -18,17 +18,43 @@ define void @early_exit_zext_trip_count(ptr %p) {
; CHECK: preheader:
; CHECK-NEXT: [[EXIT_32:%.*]] = add nsw i32 [[LEN]], -1
; CHECK-NEXT: [[EXIT:%.*]] = zext i32 [[EXIT_32]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i64 [[EXIT]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
+; CHECK-NEXT: [[IV_NEXT:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY_INTERIM:%.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[IV_NEXT]]
+; CHECK-NEXT: br i1 [[TMP4]], label [[VECTOR_EARLY_EXIT:%.*]], label [[VECTOR_BODY_INTERIM]]
+; CHECK: vector.body.interim:
+; CHECK-NEXT: br i1 [[TMP5]], label [[LATCH:%.*]], label [[LOOP]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[IV_NEXT]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[RET_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: vector.early.exit:
+; CHECK-NEXT: br label [[DEOPT:%.*]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT]], [[LATCH]] ], [ 0, [[PREHEADER]] ]
+; CHECK-NEXT: br label [[LOOP2:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ], [ 0, [[PREHEADER]] ]
-; CHECK-NEXT: [[ELEM_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 [[IV]]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT1:%.*]], [[LATCH1:%.*]] ], [ [[IV]], [[SCALAR_PH]] ]
+; CHECK-NEXT: [[ELEM_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 [[IV1]]
; CHECK-NEXT: [[ELEM:%.*]] = load i8, ptr [[ELEM_PTR]], align 1
; CHECK-NEXT: [[IS_TARGET:%.*]] = icmp eq i8 [[ELEM]], 0
-; CHECK-NEXT: br i1 [[IS_TARGET]], label [[DEOPT:%.*]], label [[LATCH]]
+; CHECK-NEXT: br i1 [[IS_TARGET]], label [[DEOPT]], label [[LATCH1]]
; CHECK: latch:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp ult i64 [[IV]], [[EXIT]]
-; CHECK-NEXT: br i1 [[LOOP_COND]], label [[LOOP]], label [[RET_LOOPEXIT:%.*]]
+; CHECK-NEXT: [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp ult i64 [[IV1]], [[EXIT]]
+; CHECK-NEXT: br i1 [[LOOP_COND]], label [[LOOP2]], label [[RET_LOOPEXIT]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: ret.loopexit:
; CHECK-NEXT: br label [[RET]]
; CHECK: ret:
@@ -76,4 +102,8 @@ declare void @deopt()
!0 = !{i32 0, i32 1000}
;.
; CHECK: [[RNG0]] = !{i32 0, i32 1000}
+; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META2]]}
;.
More information about the llvm-commits
mailing list