[llvm] [SCEV] Improve WrapPredicate nusw implication (PR #219462)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 28 06:08:05 PDT 2026
https://github.com/artagnon created https://github.com/llvm/llvm-project/pull/219462
The conditions are too strong at the moment, as a consequence of sharing it with the nssw codepath.
Proof: https://alive2.llvm.org/ce/z/7ReJnh
>From f1c17af26d3f5a52cfe38bcb7471e084bc8fcbd9 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 28 Aug 2026 13:22:45 +0100
Subject: [PATCH] [SCEV] Improve WrapPredicate nusw implication
The conditions are too strong at the moment, as a consequence of sharing
it with the nssw codepath.
Proof: https://alive2.llvm.org/ce/z/7ReJnh
---
llvm/lib/Analysis/ScalarEvolution.cpp | 30 +++----
.../bounded-access-pattern.ll | 2 +-
.../inbounds-gep-in-predicated-blocks.ll | 6 +-
.../LoopAccessAnalysis/nusw-predicates.ll | 2 -
...untime-checks-after-dependence-analysis.ll | 1 -
...time-checks-non-addrec-after-predicates.ll | 6 +-
.../AArch64/conditional-branches-cost.ll | 24 +-----
.../LoopVectorize/AArch64/induction-costs.ll | 29 +++----
...terleave-group-requires-scalar-epilogue.ll | 7 +-
...-narrow-interleave-to-widen-memory-cost.ll | 29 +------
...sform-narrow-interleave-to-widen-memory.ll | 18 +----
.../truncate-to-minimal-bitwidth-cost.ll | 2 +-
.../LoopVectorize/X86/cost-model.ll | 9 +--
.../LoopVectorize/X86/induction-step.ll | 9 +--
.../LoopVectorize/X86/interleave-cost.ll | 51 +-----------
...d-inductions-vs-first-order-recurrences.ll | 4 +-
.../LoopVectorize/predicated-inductions.ll | 80 +++++++++++++++----
.../runtime-check-small-bounded-ranges.ll | 16 ++--
...ive-path-inner-loop-with-runtime-checks.ll | 22 ++---
19 files changed, 128 insertions(+), 219 deletions(-)
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 7aaad7a89d11c..31aca658c2b56 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -15384,23 +15384,23 @@ bool SCEVWrapPredicate::implies(const SCEVPredicate *N,
const SCEV *Step = AR->getStepRecurrence(SE);
const SCEV *OpStep = Op->AR->getStepRecurrence(SE);
- if (!SE.isKnownPositive(Step) || !SE.isKnownPositive(OpStep))
- return false;
- // If both steps are positive, this implies N, if N's start and step are
- // ULE/SLE (for NSUW/NSSW) than this'.
+ // The widening is only for type mismatches. The kind of widening does not
+ // matter.
Type *WiderTy = SE.getWiderType(Step->getType(), OpStep->getType());
- Step = SE.getNoopOrZeroExtend(Step, WiderTy);
- OpStep = SE.getNoopOrZeroExtend(OpStep, WiderTy);
-
- bool IsNUW = Flags == SCEVWrapPredicate::IncrementNUSW;
- OpStart = IsNUW ? SE.getNoopOrZeroExtend(OpStart, WiderTy)
- : SE.getNoopOrSignExtend(OpStart, WiderTy);
- Start = IsNUW ? SE.getNoopOrZeroExtend(Start, WiderTy)
- : SE.getNoopOrSignExtend(Start, WiderTy);
- CmpInst::Predicate Pred = IsNUW ? CmpInst::ICMP_ULE : CmpInst::ICMP_SLE;
- return SE.isKnownPredicate(Pred, OpStep, Step) &&
- SE.isKnownPredicate(Pred, OpStart, Start);
+ Step = SE.getNoopOrAnyExtend(Step, WiderTy);
+ Start = SE.getNoopOrZeroExtend(Start, WiderTy);
+
+ bool Cond = (SE.isKnownNonNegative(Step) && SE.isKnownNonNegative(OpStep)) ||
+ SE.isKnownPredicate(CmpInst::ICMP_SGT, OpStep, Step);
+
+ // Extra conditions for nssw.
+ Cond &= Flags == SCEVWrapPredicate::IncrementNSSW
+ ? SE.isKnownPredicate(CmpInst::ICMP_SLE, OpStart, Start) &&
+ SE.isKnownPredicate(CmpInst::ICMP_SLE, OpStep, Step)
+ : true;
+
+ return Cond;
}
bool SCEVWrapPredicate::isAlwaysTrue() const {
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/bounded-access-pattern.ll b/llvm/test/Analysis/LoopAccessAnalysis/bounded-access-pattern.ll
index 020d808f7b503..c0ea454c192a5 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/bounded-access-pattern.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/bounded-access-pattern.ll
@@ -257,7 +257,7 @@ define void @bounded_offset_load(ptr %a) {
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
-; CHECK-NEXT: {1,+,1}<%loop> Added Flags: <nusw>
+; CHECK-NEXT: {0,+,1}<%loop> Added Flags: <nusw>
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
; CHECK-NEXT: [PSE] %gep.load = getelementptr inbounds i32, ptr %a, i64 %idx.load:
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/inbounds-gep-in-predicated-blocks.ll b/llvm/test/Analysis/LoopAccessAnalysis/inbounds-gep-in-predicated-blocks.ll
index aeaf59c7aea44..2a574dd904a34 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/inbounds-gep-in-predicated-blocks.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/inbounds-gep-in-predicated-blocks.ll
@@ -26,7 +26,7 @@ define void @test_inbounds_gep_used_in_predicated_block(ptr %A, i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
-; CHECK-NEXT: {%A,+,4611686018427387906}<%loop.header> Added Flags: <nusw>
+; CHECK-NEXT: {%A,+,4611686018427387905}<%loop.header> Added Flags: <nusw>
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
;
@@ -75,7 +75,7 @@ define void @test_inbounds_gep_used_in_predicated_block_stored_value_operand(ptr
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were found in loop.
; CHECK-NEXT: SCEV assumptions:
-; CHECK-NEXT: {%A,+,4611686018427387906}<%loop.header> Added Flags: <nusw>
+; CHECK-NEXT: {%A,+,4611686018427387905}<%loop.header> Added Flags: <nusw>
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
;
@@ -126,7 +126,7 @@ define void @test_inbounds_gep_used_in_predicated_block_non_memop_user(ptr %A, i
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
-; CHECK-NEXT: {%A,+,4611686018427387906}<%loop.header> Added Flags: <nusw>
+; CHECK-NEXT: {%A,+,4611686018427387905}<%loop.header> Added Flags: <nusw>
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/nusw-predicates.ll b/llvm/test/Analysis/LoopAccessAnalysis/nusw-predicates.ll
index 87b25140949b9..73836cf0a9f35 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/nusw-predicates.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/nusw-predicates.ll
@@ -100,7 +100,6 @@ define void @int_and_multiple_pointer_predicates(ptr %v, ptr %w, i32 %N) {
; CHECK-NEXT: SCEV assumptions:
; CHECK-NEXT: {0,+,1}<%loop> Added Flags: <nusw>
; CHECK-NEXT: {%v,+,4}<%loop> Added Flags: <nusw>
-; CHECK-NEXT: {%w,+,4}<%loop> Added Flags: <nusw>
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
;
@@ -175,7 +174,6 @@ define void @wider_i32_nusw_does_not_imply_narrower_i4_nusw(ptr %dst, i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
-; CHECK-NEXT: {1,+,1}<%loop> Added Flags: <nusw>
; CHECK-NEXT: {0,+,1}<%loop> Added Flags: <nusw>
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
index 88b16d0f32534..2f99cd97db866 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
@@ -352,7 +352,6 @@ define void @retry_after_dep_check_with_unknown_offset(ptr %A, i32 %offset) {
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
; CHECK-NEXT: {(100 + %A),+,8}<%loop> Added Flags: <nusw>
-; CHECK-NEXT: {(100 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A),+,8}<%loop> Added Flags: <nusw>
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
;
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-non-addrec-after-predicates.ll b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-non-addrec-after-predicates.ll
index 86125b1e55ace..670054ca555c5 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-non-addrec-after-predicates.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-non-addrec-after-predicates.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck %s
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
@@ -18,13 +19,12 @@ define void @retry_do_not_cast_non_addrec(ptr %p) {
; CHECK-NEXT: SCEV assumptions:
; CHECK-NEXT: {0,+,16}<%loop> Added Flags: <nusw>
; CHECK-NEXT: {%p,+,16}<%loop> Added Flags: <nusw>
-; CHECK-NEXT: {(1 + %p),+,16}<%loop> Added Flags: <nusw>
; CHECK-EMPTY:
; CHECK-NEXT: Expressions re-written:
-; CHECK-NEXT: [PSE] %gep.0 = getelementptr i8, ptr %p, i64 %idx.0:
+; CHECK-NEXT: [PSE] %gep.0 = getelementptr i8, ptr %p, i64 %idx.0:
; CHECK-NEXT: ((zext i32 %iv to i64) + %p)
; CHECK-NEXT: --> {%p,+,16}<%loop>
-; CHECK-NEXT: [PSE] %gep.1 = getelementptr i8, ptr %p, i64 %idx.1:
+; CHECK-NEXT: [PSE] %gep.1 = getelementptr i8, ptr %p, i64 %idx.1:
; CHECK-NEXT: ((zext i32 (1 + %iv)<nsw> to i64) + %p)
; CHECK-NEXT: --> {(1 + %p),+,16}<%loop>
;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 87d6325d640e5..f29a5ef6181cb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -670,17 +670,7 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
; DEFAULT-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
; DEFAULT-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
; DEFAULT-NEXT: [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[DST]]
-; DEFAULT-NEXT: [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
-; DEFAULT-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4
-; DEFAULT-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; DEFAULT-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; DEFAULT-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
-; DEFAULT-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT]]
-; DEFAULT-NEXT: [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP4]]
-; DEFAULT-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
-; DEFAULT-NEXT: [[TMP13:%.*]] = or i1 [[TMP4]], [[TMP8]]
-; DEFAULT-NEXT: [[TMP14:%.*]] = or i1 [[TMP13]], [[TMP12]]
+; DEFAULT-NEXT: [[TMP14:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
; DEFAULT-NEXT: br i1 [[TMP14]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 7
@@ -822,17 +812,7 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
; PRED-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
; PRED-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
; PRED-NEXT: [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[DST]]
-; PRED-NEXT: [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
-; PRED-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4
-; PRED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; PRED-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; PRED-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
-; PRED-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; PRED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT]]
-; PRED-NEXT: [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP4]]
-; PRED-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
-; PRED-NEXT: [[TMP13:%.*]] = or i1 [[TMP4]], [[TMP8]]
-; PRED-NEXT: [[TMP14:%.*]] = or i1 [[TMP13]], [[TMP12]]
+; PRED-NEXT: [[TMP14:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
; PRED-NEXT: br i1 [[TMP14]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; PRED: [[VECTOR_PH]]:
; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 [[TMP0]])
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index 0817c7622641e..4e8841c6a85d4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -529,16 +529,6 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[N]], i64 0)
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[SMIN]]
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 -8
-; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SRC]]
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP2]] to i32
; CHECK-NEXT: [[MUL4:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 2, i32 [[TMP9]])
; CHECK-NEXT: [[MUL_RESULT5:%.*]] = extractvalue { i32, i1 } [[MUL4]], 0
@@ -547,13 +537,14 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP10]], [[MUL_OVERFLOW6]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp ugt i64 [[TMP2]], 4294967295
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
+; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
; CHECK-NEXT: [[TMP14:%.*]] = sub i64 0, [[MUL_RESULT]]
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP14]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp ugt ptr [[TMP15]], [[BASE]]
; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP16]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP5]], [[TMP8]]
-; CHECK-NEXT: [[TMP19:%.*]] = or i1 [[TMP18]], [[TMP13]]
-; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP19]], [[TMP17]]
+; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP13]], [[TMP17]]
; CHECK-NEXT: br i1 [[TMP20]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SCEVGEP10:%.*]] = getelementptr i8, ptr [[SRC]], i64 -8
@@ -701,14 +692,14 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK-NEXT: [[CMP_N29:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC23]]
; CHECK-NEXT: br i1 [[CMP_N29]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL23:%.*]] = phi i64 [ [[TMP87]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP26]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL24:%.*]] = phi i64 [ [[TMP88]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP27]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL25:%.*]] = phi i32 [ [[TMP90]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP29]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL22:%.*]] = phi i64 [ [[TMP87]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP26]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL23:%.*]] = phi i64 [ [[TMP88]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP27]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL24:%.*]] = phi i32 [ [[TMP90]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP29]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL23]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[COUNT:%.*]] = phi i64 [ [[BC_RESUME_VAL24]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[COUNT_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[IV_I32:%.*]] = phi i32 [ [[BC_RESUME_VAL25]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_I32_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL22]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[COUNT:%.*]] = phi i64 [ [[BC_RESUME_VAL23]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[COUNT_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_I32:%.*]] = phi i32 [ [[BC_RESUME_VAL24]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_I32_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr i8, ptr [[GEP_1]], i64 -8
; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_2]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-group-requires-scalar-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-group-requires-scalar-epilogue.ll
index 8ab8804b1e118..0001e48b2c216 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-group-requires-scalar-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-group-requires-scalar-epilogue.ll
@@ -18,12 +18,7 @@ define void @interleave_group_exit_in_header(i64 %n, ptr %dst) {
; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr [[TMP1]], [[DST]]
-; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[TMP4]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP5]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP3]], [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
index fbf352c2ff56f..da93b23a27169 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
@@ -155,12 +155,7 @@ define void @test_interleave_store_one_constant(ptr noalias %src, ptr noalias %d
; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[DST]]
-; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP4]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK5:%.*]] = icmp ult i64 [[TMP0]], 8
@@ -283,20 +278,7 @@ define void @single_fmul_used_by_each_member(ptr noalias %A, ptr noalias %B, ptr
; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[B]]
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[C]], i64 8
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP4]]
-; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[C]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP15:%.*]] = icmp ult ptr [[TMP14]], [[C]]
-; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP4]], [[TMP8]]
-; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP17]], [[TMP12]]
-; CHECK-NEXT: [[TMP19:%.*]] = or i1 [[TMP18]], [[TMP16]]
+; CHECK-NEXT: [[TMP19:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
; CHECK-NEXT: br i1 [[TMP19]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK11:%.*]] = icmp ult i64 [[TMP0]], 8
@@ -515,12 +497,7 @@ define void @test_interleave_group_epilogue_with_preheader_phi(ptr %src, ptr %ds
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp ult ptr [[TMP8]], [[DST]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP9]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP12:%.*]] = icmp ult ptr [[TMP11]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP12]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[IDENT_CHECK]], [[TMP10]]
-; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP14]], [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[IDENT_CHECK]], [[TMP10]]
; CHECK-NEXT: br i1 [[TMP15]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK5:%.*]] = icmp ult i64 [[TMP2]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
index f9914ff508423..aed7c31e06dcd 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
@@ -104,7 +104,7 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 26)
+; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 18)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], [[UMAX]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
@@ -113,12 +113,7 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[DST]]
-; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP1]]
@@ -161,7 +156,7 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; EPILOGUE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; EPILOGUE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; EPILOGUE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; EPILOGUE-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 28)
+; EPILOGUE-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 20)
; EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], [[UMAX]]
; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; EPILOGUE: [[VECTOR_SCEVCHECK]]:
@@ -170,12 +165,7 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; EPILOGUE-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
; EPILOGUE-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
; EPILOGUE-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[DST]]
-; EPILOGUE-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
-; EPILOGUE-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; EPILOGUE-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; EPILOGUE-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
-; EPILOGUE-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
+; EPILOGUE-NEXT: [[TMP9:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
; EPILOGUE-NEXT: br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; EPILOGUE: [[VECTOR_PH]]:
; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index 1f02c3f4906b7..3bf8ef8c61747 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -253,7 +253,7 @@ define void @test_minbws_for_trunc(i32 %n, ptr noalias %p1, ptr noalias %p2) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P2]], i64 8160
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P2]], i64 2040
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[P2]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 2c740670be578..3c7fb470c67e6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -454,21 +454,16 @@ define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
; CHECK-NEXT: [[TMP1:%.*]] = freeze i64 [[TMP0]]
; CHECK-NEXT: [[UMIN10:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[A]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 [[UMIN10]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], 24
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], 18
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[B]], i64 1)
; CHECK-NEXT: [[TMP4:%.*]] = freeze i64 [[TMP3]]
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP4]], i64 [[A]])
; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[UMIN]] to i32
-; CHECK-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 1
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMIN]], 4294967295
-; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp slt i32 [[TMP5]], 0
; CHECK-NEXT: [[TMP12:%.*]] = icmp ugt i64 [[UMIN]], 4294967295
-; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP9]], [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP11]], [[TMP12]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
index 55365a9d3ed0a..ba5e5017e0899 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
@@ -148,7 +148,7 @@ define void @canonical_iv_matches_user_iv(i64 %n, ptr %A, ptr %B) {
; CHECK-LABEL: define void @canonical_iv_matches_user_iv(
; CHECK-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 6
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
@@ -158,12 +158,7 @@ define void @canonical_iv_matches_user_iv(i64 %n, ptr %A, ptr %B) {
; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr [[TMP1]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 24
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[TMP4]], [[SCEVGEP1]]
-; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP5]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP3]], [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[A]], i64 8
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
index aa5a0f5be24be..83f6fe0c46737 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
@@ -179,7 +179,7 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = lshr i64 [[N]], 3
; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 28
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 12
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[N]], 3
@@ -188,54 +188,7 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[A]]
-; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 4
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[A]], i64 8
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP10:%.*]] = icmp ult ptr [[TMP9]], [[SCEVGEP4]]
-; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP10]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[A]], i64 12
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SCEVGEP8]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP13:%.*]] = icmp ult ptr [[TMP12]], [[SCEVGEP8]]
-; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP13]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP12:%.*]] = getelementptr i8, ptr [[A]], i64 16
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[SCEVGEP12]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP16:%.*]] = icmp ult ptr [[TMP15]], [[SCEVGEP12]]
-; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP16]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP16:%.*]] = getelementptr i8, ptr [[A]], i64 20
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[SCEVGEP16]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP19:%.*]] = icmp ult ptr [[TMP18]], [[SCEVGEP16]]
-; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP19]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP20:%.*]] = getelementptr i8, ptr [[A]], i64 24
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[SCEVGEP20]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP22:%.*]] = icmp ult ptr [[TMP21]], [[SCEVGEP20]]
-; CHECK-NEXT: [[TMP23:%.*]] = or i1 [[TMP22]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP24:%.*]] = getelementptr i8, ptr [[A]], i64 28
-; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[SCEVGEP24]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP25:%.*]] = icmp ult ptr [[TMP24]], [[SCEVGEP24]]
-; CHECK-NEXT: [[TMP26:%.*]] = or i1 [[TMP25]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[SCEVGEP28:%.*]] = getelementptr i8, ptr [[B]], i64 4
-; CHECK-NEXT: [[MUL29:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT30:%.*]] = extractvalue { i64, i1 } [[MUL29]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW31:%.*]] = extractvalue { i64, i1 } [[MUL29]], 1
-; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[SCEVGEP28]], i64 [[MUL_RESULT30]]
-; CHECK-NEXT: [[TMP28:%.*]] = icmp ult ptr [[TMP27]], [[SCEVGEP28]]
-; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP28]], [[MUL_OVERFLOW31]]
-; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[B]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP31:%.*]] = icmp ult ptr [[TMP30]], [[B]]
-; CHECK-NEXT: [[TMP32:%.*]] = or i1 [[TMP31]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP33:%.*]] = or i1 [[TMP5]], [[TMP8]]
-; CHECK-NEXT: [[TMP34:%.*]] = or i1 [[TMP33]], [[TMP11]]
-; CHECK-NEXT: [[TMP35:%.*]] = or i1 [[TMP34]], [[TMP14]]
-; CHECK-NEXT: [[TMP36:%.*]] = or i1 [[TMP35]], [[TMP17]]
-; CHECK-NEXT: [[TMP37:%.*]] = or i1 [[TMP36]], [[TMP20]]
-; CHECK-NEXT: [[TMP38:%.*]] = or i1 [[TMP37]], [[TMP23]]
-; CHECK-NEXT: [[TMP39:%.*]] = or i1 [[TMP38]], [[TMP26]]
-; CHECK-NEXT: [[TMP40:%.*]] = or i1 [[TMP39]], [[TMP29]]
-; CHECK-NEXT: [[TMP41:%.*]] = or i1 [[TMP40]], [[TMP32]]
+; CHECK-NEXT: [[TMP41:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
; CHECK-NEXT: br i1 [[TMP41]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP42:%.*]] = lshr i64 [[N]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-inductions-vs-first-order-recurrences.ll b/llvm/test/Transforms/LoopVectorize/predicated-inductions-vs-first-order-recurrences.ll
index 36119d8d49d96..367e12f918f6a 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-inductions-vs-first-order-recurrences.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-inductions-vs-first-order-recurrences.ll
@@ -434,6 +434,8 @@ define i64 @for_and_ind_liveout_gep(ptr %dst, i64 %n) {
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP4]], i64 3
+; CHECK-NEXT: [[TMP11:%.*]] = sub nuw i64 [[N_VEC]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 1, [[TMP11]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -453,7 +455,7 @@ define i64 @for_and_ind_liveout_gep(ptr %dst, i64 %n) {
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RESULT:%.*]] = phi i64 [ [[EXT]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RESULT:%.*]] = phi i64 [ [[EXT]], %[[LOOP]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RESULT]]
;
; THRESHOLD-LABEL: define i64 @for_and_ind_liveout_gep(
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll b/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
index 80d32ca536ced..d72314c7b3d43 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
@@ -43,6 +43,8 @@ define i64 @predicated_iv_with_liveout(ptr %dst, i64 %n) {
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP4]], i64 3
+; CHECK-NEXT: [[TMP11:%.*]] = sub nuw i64 [[N_VEC]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 1, [[TMP11]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -62,7 +64,7 @@ define i64 @predicated_iv_with_liveout(ptr %dst, i64 %n) {
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RESULT:%.*]] = phi i64 [ [[EXT]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RESULT:%.*]] = phi i64 [ [[EXT]], %[[LOOP]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RESULT]]
;
; THRESHOLD0-LABEL: define i64 @predicated_iv_with_liveout(
@@ -118,6 +120,8 @@ define i64 @predicated_iv_with_liveout(ptr %dst, i64 %n) {
; THRESHOLD1-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; THRESHOLD1: [[MIDDLE_BLOCK]]:
; THRESHOLD1-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP4]], i64 3
+; THRESHOLD1-NEXT: [[TMP11:%.*]] = sub nuw i64 [[N_VEC]], 1
+; THRESHOLD1-NEXT: [[TMP12:%.*]] = add i64 1, [[TMP11]]
; THRESHOLD1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
; THRESHOLD1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; THRESHOLD1: [[SCALAR_PH]]:
@@ -137,7 +141,7 @@ define i64 @predicated_iv_with_liveout(ptr %dst, i64 %n) {
; THRESHOLD1-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; THRESHOLD1-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; THRESHOLD1: [[EXIT]]:
-; THRESHOLD1-NEXT: [[RESULT:%.*]] = phi i64 [ [[EXT]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
+; THRESHOLD1-NEXT: [[RESULT:%.*]] = phi i64 [ [[EXT]], %[[LOOP]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
; THRESHOLD1-NEXT: ret i64 [[RESULT]]
;
entry:
@@ -967,17 +971,11 @@ define void @two_used_predicated_ivs(ptr %dst1, ptr %dst2, i64 %n) {
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i8
-; CHECK-NEXT: [[MUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 9, i8 [[TMP1]])
-; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i8, i1 } [[MUL]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[TMP0]], 255
-; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[MUL_OVERFLOW]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP0]] to i7
; CHECK-NEXT: [[MUL1:%.*]] = call { i7, i1 } @llvm.umul.with.overflow.i7(i7 5, i7 [[TMP4]])
; CHECK-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i7, i1 } [[MUL1]], 1
; CHECK-NEXT: [[TMP5:%.*]] = icmp ugt i64 [[TMP0]], 127
-; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[MUL_OVERFLOW3]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP3]], [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[MUL_OVERFLOW3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[DST24]], [[DST15]]
@@ -1011,13 +1009,13 @@ define void @two_used_predicated_ivs(ptr %dst1, ptr %dst2, i64 %n) {
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL8:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL9:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[PRED_IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL8]], %[[SCALAR_PH]] ], [ [[PRED_NEXT1:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[PRED_IV2:%.*]] = phi i32 [ [[BC_RESUME_VAL9]], %[[SCALAR_PH]] ], [ [[PRED_NEXT2:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PRED_IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[PRED_NEXT1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PRED_IV2:%.*]] = phi i32 [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ], [ [[PRED_NEXT2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[MASKED1:%.*]] = and i32 [[PRED_IV1]], 255
; CHECK-NEXT: [[PRED_NEXT1]] = add nuw nsw i32 [[MASKED1]], 9
@@ -1057,11 +1055,59 @@ define void @two_used_predicated_ivs(ptr %dst1, ptr %dst2, i64 %n) {
; THRESHOLD1-LABEL: define void @two_used_predicated_ivs(
; THRESHOLD1-SAME: ptr [[DST1:%.*]], ptr [[DST2:%.*]], i64 [[N:%.*]]) {
; THRESHOLD1-NEXT: [[ENTRY:.*]]:
+; THRESHOLD1-NEXT: [[DST12:%.*]] = ptrtoaddr ptr [[DST1]] to i64
+; THRESHOLD1-NEXT: [[DST21:%.*]] = ptrtoaddr ptr [[DST2]] to i64
+; THRESHOLD1-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; THRESHOLD1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; THRESHOLD1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; THRESHOLD1: [[VECTOR_SCEVCHECK]]:
+; THRESHOLD1-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; THRESHOLD1-NEXT: [[TMP1:%.*]] = add nsw i64 [[SMAX]], -1
+; THRESHOLD1-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i7
+; THRESHOLD1-NEXT: [[MUL:%.*]] = call { i7, i1 } @llvm.umul.with.overflow.i7(i7 5, i7 [[TMP2]])
+; THRESHOLD1-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i7, i1 } [[MUL]], 1
+; THRESHOLD1-NEXT: [[TMP3:%.*]] = icmp ugt i64 [[TMP1]], 127
+; THRESHOLD1-NEXT: [[TMP4:%.*]] = or i1 [[MUL_OVERFLOW]], [[TMP3]]
+; THRESHOLD1-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; THRESHOLD1: [[VECTOR_MEMCHECK]]:
+; THRESHOLD1-NEXT: [[TMP5:%.*]] = sub i64 [[DST21]], [[DST12]]
+; THRESHOLD1-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; THRESHOLD1-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], 15
+; THRESHOLD1-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; THRESHOLD1: [[VECTOR_PH]]:
+; THRESHOLD1-NEXT: [[TMP7:%.*]] = and i64 [[TMP0]], 3
+; THRESHOLD1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP7]]
+; THRESHOLD1-NEXT: [[TMP8:%.*]] = trunc i64 [[N_VEC]] to i32
+; THRESHOLD1-NEXT: [[TMP9:%.*]] = mul i32 [[TMP8]], 9
+; THRESHOLD1-NEXT: [[TMP10:%.*]] = mul i32 [[TMP8]], 5
; THRESHOLD1-NEXT: br label %[[LOOP:.*]]
; THRESHOLD1: [[LOOP]]:
-; THRESHOLD1-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; THRESHOLD1-NEXT: [[PRED_IV1:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[PRED_NEXT1:%.*]], %[[LOOP]] ]
-; THRESHOLD1-NEXT: [[PRED_IV2:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[PRED_NEXT2:%.*]], %[[LOOP]] ]
+; THRESHOLD1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD1-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 9, i32 18, i32 27>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD1-NEXT: [[VEC_IND3:%.*]] = phi <4 x i32> [ <i32 0, i32 5, i32 10, i32 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], %[[LOOP]] ]
+; THRESHOLD1-NEXT: [[TMP11:%.*]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 9)
+; THRESHOLD1-NEXT: [[TMP12:%.*]] = add nuw nsw <4 x i32> [[VEC_IND3]], splat (i32 5)
+; THRESHOLD1-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[DST1]], i64 [[INDEX]]
+; THRESHOLD1-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP13]], align 4
+; THRESHOLD1-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[DST2]], i64 [[INDEX]]
+; THRESHOLD1-NEXT: store <4 x i32> [[TMP12]], ptr [[TMP14]], align 4
+; THRESHOLD1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; THRESHOLD1-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 36)
+; THRESHOLD1-NEXT: [[VEC_IND_NEXT4]] = add nuw nsw <4 x i32> [[VEC_IND3]], splat (i32 20)
+; THRESHOLD1-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; THRESHOLD1-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; THRESHOLD1: [[MIDDLE_BLOCK]]:
+; THRESHOLD1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; THRESHOLD1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; THRESHOLD1: [[SCALAR_PH]]:
+; THRESHOLD1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; THRESHOLD1-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; THRESHOLD1-NEXT: [[BC_RESUME_VAL6:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; THRESHOLD1-NEXT: br label %[[LOOP1:.*]]
+; THRESHOLD1: [[LOOP1]]:
+; THRESHOLD1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; THRESHOLD1-NEXT: [[PRED_IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[PRED_NEXT1:%.*]], %[[LOOP1]] ]
+; THRESHOLD1-NEXT: [[PRED_IV2:%.*]] = phi i32 [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ], [ [[PRED_NEXT2:%.*]], %[[LOOP1]] ]
; THRESHOLD1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; THRESHOLD1-NEXT: [[MASKED1:%.*]] = and i32 [[PRED_IV1]], 255
; THRESHOLD1-NEXT: [[PRED_NEXT1]] = add nuw nsw i32 [[MASKED1]], 9
@@ -1072,7 +1118,7 @@ define void @two_used_predicated_ivs(ptr %dst1, ptr %dst2, i64 %n) {
; THRESHOLD1-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[DST2]], i64 [[IV]]
; THRESHOLD1-NEXT: store i32 [[PRED_NEXT2]], ptr [[GEP2]], align 4
; THRESHOLD1-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; THRESHOLD1-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; THRESHOLD1-NEXT: br i1 [[CMP]], label %[[LOOP1]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
; THRESHOLD1: [[EXIT]]:
; THRESHOLD1-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-small-bounded-ranges.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-small-bounded-ranges.ll
index 784dcab3e7847..a1d5b62507449 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check-small-bounded-ranges.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-small-bounded-ranges.ll
@@ -498,14 +498,8 @@ define void @wider_nusw_does_not_imply_narrower(ptr %dst, i64 %n) {
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP4:%.*]] = add i32 1, [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP4]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i64 [[TMP2]], 4294967295
-; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[TMP2]], 15
-; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT: br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP1]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
@@ -530,12 +524,12 @@ define void @wider_nusw_does_not_imply_narrower(ptr %dst, i64 %n) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[TMP19:%.*]] = phi i64 [ [[TMP17]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT: [[TMP20:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: [[TMP18:%.*]] = phi i64 [ [[TMP17]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: [[TMP19:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV_ZEXT:%.*]] = phi i64 [ [[TMP19]], %[[SCALAR_PH]] ], [ [[IV_NEXT_ZEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP20]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_ZEXT:%.*]] = phi i64 [ [[TMP18]], %[[SCALAR_PH]] ], [ [[IV_NEXT_ZEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP19]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_MOD:%.*]] = and i32 [[IV]], 15
; CHECK-NEXT: [[IV_MOD_ZEXT:%.*]] = zext nneg i32 [[IV_MOD]] to i64
; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 [[IV_MOD_ZEXT]]
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
index b17f36a7f5ae3..bc1ad2efb9bf5 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
@@ -12,9 +12,7 @@ define void @expand(ptr %src, ptr %dst, i64 %0) {
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[SMAX]], -1
; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], [[TMP0]]
; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP0]], 4
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 8
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]]
; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 8
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP0]], 1
; CHECK-NEXT: [[SMAX6:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 1000)
@@ -32,24 +30,20 @@ define void @expand(ptr %src, ptr %dst, i64 %0) {
; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP3]])
; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp ult ptr [[TMP14]], [[SCEVGEP1]]
-; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP12]], [[TMP16]]
+; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP7]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP5]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP5]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP8]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP8]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[SRC]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT: [[TMP16:%.*]] = load double, ptr [[SRC]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT]], <i64 0, i64 1, i64 2, i64 3>
@@ -66,10 +60,10 @@ define void @expand(ptr %src, ptr %dst, i64 %0) {
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP25]]
; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP20]], i64 3
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP27]]
-; CHECK-NEXT: store double [[TMP22]], ptr [[TMP31]], align 8, !alias.scope [[META3]]
-; CHECK-NEXT: store double [[TMP22]], ptr [[TMP26]], align 8, !alias.scope [[META3]]
-; CHECK-NEXT: store double [[TMP22]], ptr [[TMP33]], align 8, !alias.scope [[META3]]
-; CHECK-NEXT: store double [[TMP22]], ptr [[TMP28]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT: store double [[TMP16]], ptr [[TMP31]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT: store double [[TMP16]], ptr [[TMP26]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT: store double [[TMP16]], ptr [[TMP33]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT: store double [[TMP16]], ptr [[TMP28]], align 8, !alias.scope [[META3]]
; CHECK-NEXT: [[TMP29:%.*]] = or disjoint <4 x i64> [[TMP20]], splat (i64 1)
; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[TMP29]], i64 0
; CHECK-NEXT: [[TMP41:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP30]]
More information about the llvm-commits
mailing list