[llvm] cb71fef - [VPlan] Expand SCEV min/max via scalar intrinsic VPInstructions. (#207842)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 03:39:40 PDT 2026
Author: Florian Hahn
Date: 2026-07-13T11:39:35+01:00
New Revision: cb71fefb2258fdd00747043a9c322f95d684b000
URL: https://github.com/llvm/llvm-project/commit/cb71fefb2258fdd00747043a9c322f95d684b000
DIFF: https://github.com/llvm/llvm-project/commit/cb71fefb2258fdd00747043a9c322f95d684b000.diff
LOG: [VPlan] Expand SCEV min/max via scalar intrinsic VPInstructions. (#207842)
Use the VPInstruction::Intrinsic opcode to expand min/max SCEV
expressions to LLVM min/max intrinsics as VPInstructions.
The operands are chained in reverse order, matching SCEVExpander's
expansion of min/max expressions.
This helps to clean up another set of duplicated vscale calls.
Depends on https://github.com/llvm/llvm-project/pull/207541
Depends on https://github.com/llvm/llvm-project/pull/207836
PR: https://github.com/llvm/llvm-project/pull/207842
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
llvm/test/Transforms/LoopVectorize/AArch64/fold-tail-low-trip-count.ll
llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
llvm/test/Transforms/LoopVectorize/RISCV/conditional-scalar-assignment.ll
llvm/test/Transforms/LoopVectorize/RISCV/dbg-tail-folding-by-evl.ll
llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll
llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv32.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-masked-loadstore.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll
llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
llvm/test/Transforms/LoopVectorize/cast-induction.ll
llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
llvm/test/Transforms/LoopVectorize/find-last.ll
llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
llvm/test/Transforms/LoopVectorize/induction-ptrcasts.ll
llvm/test/Transforms/LoopVectorize/invalidate-scev-at-scope-after-vectorization.ll
llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
llvm/test/Transforms/LoopVectorize/pr125278.ll
llvm/test/Transforms/LoopVectorize/pr37248.ll
llvm/test/Transforms/LoopVectorize/pr38697.ll
llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
llvm/test/Transforms/LoopVectorize/replace-first-order-recurrence-by-versioned-iv.ll
llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll
llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 205fc18a86c78..7b4ffce87434c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -842,6 +842,44 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
}
return Builder.createScalarCast(Opcode, Op, S->getType(), DL);
}
+ case scUMaxExpr:
+ case scSMaxExpr:
+ case scUMinExpr:
+ case scSMinExpr: {
+ auto *MinMax = cast<SCEVMinMaxExpr>(S);
+ Intrinsic::ID IntrinsicID;
+ switch (S->getSCEVType()) {
+ case scUMaxExpr:
+ IntrinsicID = Intrinsic::umax;
+ break;
+ case scSMaxExpr:
+ IntrinsicID = Intrinsic::smax;
+ break;
+ case scUMinExpr:
+ IntrinsicID = Intrinsic::umin;
+ break;
+ case scSMinExpr:
+ IntrinsicID = Intrinsic::smin;
+ break;
+ default:
+ llvm_unreachable("Unexpected min/max SCEV type");
+ }
+ // Chain operands in reverse order matching SCEVExpander's expansion of
+ // min/max expressions.
+ SmallVector<VPValue *, 2> Ops;
+ for (const SCEVUse &Op : reverse(MinMax->operands())) {
+ VPValue *OpV = tryToExpand(Op);
+ if (!OpV)
+ return nullptr;
+ Ops.push_back(OpV);
+ }
+ Type *ResultTy = MinMax->getType();
+ VPValue *Result = Ops.front();
+ for (VPValue *Op : drop_begin(Ops))
+ Result = Builder.createScalarIntrinsic(IntrinsicID, {Result, Op},
+ ResultTy, DL);
+ return Result;
+ }
default:
return nullptr;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index e6cb92368cbef..cc1eb9c9614f9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -1129,10 +1129,10 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; DEFAULT-NEXT: [[B3:%.*]] = ptrtoaddr ptr [[B]] to i64
; DEFAULT-NEXT: [[A2:%.*]] = ptrtoaddr ptr [[A]] to i64
; DEFAULT-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
+; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 8)
-; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
@@ -1148,8 +1148,7 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; DEFAULT-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; DEFAULT-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH1:.*]]
; DEFAULT: [[VECTOR_PH1]]:
-; DEFAULT-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP45:%.*]] = shl nuw i64 [[TMP9]], 2
+; DEFAULT-NEXT: [[TMP45:%.*]] = shl nuw i64 [[TMP1]], 2
; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP45]]
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[Y]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
index d1e9faca4e6f6..5fbaa65cea311 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
@@ -20,8 +20,7 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP8]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP10]], 1
; CHECK-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP11]], 1
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP9]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-tail-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-tail-low-trip-count.ll
index 95d348576d4e1..2ff2e9ceca16d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-tail-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-tail-low-trip-count.ll
@@ -7,7 +7,7 @@ define void @low_trip_count_small(i32 %x, ptr %dst) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[X]], i32 1)
; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SMAX]], i32 4)
-; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[SMAX]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[SMAX]] to i64
; CHECK-NEXT: [[UMIN1:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP0]], i64 4)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -79,7 +79,7 @@ define ptr @low_trip_count_small_with_live_out(i32 %x, ptr %dst) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[X]], i32 1)
; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SMAX]], i32 4)
-; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[SMAX]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[SMAX]] to i64
; CHECK-NEXT: [[UMIN1:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP0]], i64 4)
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll
index 087fe17540576..821f29c1ed811 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll
@@ -69,7 +69,7 @@ define i40 @partial_reduce_not_known_factor(i32 %a, i32 %b, i16 %N) {
; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i16 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[SMAX:%.*]] = call i16 @llvm.smax.i16(i16 [[N]], i16 0)
-; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i16 [[SMAX]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = zext i16 [[SMAX]] to i32
; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[TMP0]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
index f4fcc6bf5ec02..44306ec3c38d9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
@@ -30,8 +30,7 @@ define void @multiple_exits_unique_exit_block(ptr %A, ptr %B, i32 %N) #0 {
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP11:%.*]] = shl nuw i32 [[TMP15]], 2
+; CHECK-NEXT: [[TMP11:%.*]] = shl nuw i32 [[TMP1]], 2
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw i32 [[TMP11]], 1
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], [[TMP8]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
@@ -101,8 +100,7 @@ define i32 @multiple_exits_multiple_exit_blocks(ptr %A, ptr %B, i32 %N) #0 {
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP11:%.*]] = shl nuw i32 [[TMP15]], 2
+; CHECK-NEXT: [[TMP11:%.*]] = shl nuw i32 [[TMP1]], 2
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw i32 [[TMP11]], 1
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], [[TMP8]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
index 886e7834d7f4b..0f4340731ee2f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
@@ -44,8 +44,7 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr
; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[CONFLICT_RDX9]], [[DIFF_CHECK10]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP17:%.*]] = shl nuw i64 [[TMP15]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = shl nuw i64 [[TMP0]], 1
; CHECK-NEXT: [[TMP16:%.*]] = shl nuw i64 [[TMP17]], 1
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP16]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
index aeadbf7cd8f45..87e235077b90b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
@@ -66,8 +66,7 @@ define void @many_deps(ptr noalias %buckets, ptr %array, ptr %indices, ptr %othe
; NORMAL_DEP_LIMIT-NEXT: [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]]
; NORMAL_DEP_LIMIT-NEXT: br i1 [[CONFLICT_RDX9]], label [[SCALAR_PH]], label [[ENTRY:%.*]]
; NORMAL_DEP_LIMIT: vector.ph:
-; NORMAL_DEP_LIMIT-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NORMAL_DEP_LIMIT-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP4]], 2
+; NORMAL_DEP_LIMIT-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP0]], 2
; NORMAL_DEP_LIMIT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; NORMAL_DEP_LIMIT-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NORMAL_DEP_LIMIT-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 43975ee68774e..2ec48b25af5b5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -644,8 +644,7 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP17]]
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
index 77a89ca697502..02a695ff30c06 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
@@ -83,9 +83,9 @@ define void @sgt_no_loopguard(ptr noalias nocapture readonly %a, ptr noalias noc
; COMMON-LABEL: define void @sgt_no_loopguard(
; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; COMMON-NEXT: [[ENTRY:.*:]]
-; COMMON-NEXT: [[TMP0:%.*]] = add i32 [[N]], 1
; COMMON-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[N]], i32 1)
-; COMMON-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; COMMON-NEXT: [[TMP4:%.*]] = sub i32 [[N]], [[SMIN]]
+; COMMON-NEXT: [[TMP1:%.*]] = add i32 [[TMP4]], 1
; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
; COMMON: [[VECTOR_PH]]:
; COMMON-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 15
@@ -142,9 +142,9 @@ define void @sgt_extra_use_cmp(ptr noalias nocapture readonly %a, ptr noalias no
; COMMON-LABEL: define void @sgt_extra_use_cmp(
; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; COMMON-NEXT: [[ENTRY:.*:]]
-; COMMON-NEXT: [[TMP0:%.*]] = add i32 [[N]], 1
; COMMON-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[N]], i32 1)
-; COMMON-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; COMMON-NEXT: [[TMP8:%.*]] = sub i32 [[N]], [[SMIN]]
+; COMMON-NEXT: [[TMP1:%.*]] = add i32 [[TMP8]], 1
; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
; COMMON: [[VECTOR_PH]]:
; COMMON-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/conditional-scalar-assignment.ll b/llvm/test/Transforms/LoopVectorize/RISCV/conditional-scalar-assignment.ll
index 692e30912a00a..87828ae5a7480 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/conditional-scalar-assignment.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/conditional-scalar-assignment.ll
@@ -11,8 +11,7 @@ define i32 @simple_find_last_reduction(i64 %N, ptr %data, i32 %a) {
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[UMAX]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[A]], i64 0
@@ -92,8 +91,7 @@ define i32 @non_speculatable_find_last_reduction(ptr noalias %a, ptr noalias %b,
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[UMAX]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[THRESHOLD]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dbg-tail-folding-by-evl.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dbg-tail-folding-by-evl.ll
index 079d740d4aeea..3e3c6c09050b4 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dbg-tail-folding-by-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dbg-tail-folding-by-evl.ll
@@ -7,9 +7,9 @@ define void @reverse_store(ptr %a, i64 %n) !dbg !3 {
; CHECK-LABEL: define void @reverse_store(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] !dbg [[DBG3:![0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
-; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[UMIN]]
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP1]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
@@ -19,7 +19,7 @@ define void @reverse_store(ptr %a, i64 %n) !dbg !3 {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP1]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP9]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
; CHECK-NEXT: [[TMP6:%.*]] = sub nsw i64 0, [[TMP5]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index a9db4458a34ff..5ea737a3f1a66 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -150,7 +150,7 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = zext i16 [[X]] to i32
; CHECK-NEXT: [[UMAX3:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP0]], i32 111)
-; CHECK-NEXT: [[TMP1:%.*]] = sub i32 770, [[UMAX3]]
+; CHECK-NEXT: [[TMP1:%.*]] = sub nsw i32 770, [[UMAX3]]
; CHECK-NEXT: [[SMAX4:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP1]], i32 0)
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[SMAX4]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP2]], 19
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
index aeb9fb179d4ba..64d9118209ad5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
@@ -176,15 +176,14 @@ define i64 @strided_search(ptr align 8 dereferenceable(14784) %p) {
; RV64-LABEL: define i64 @strided_search(
; RV64-SAME: ptr align 8 dereferenceable(14784) [[P:%.*]]) #[[ATTR0]] {
; RV64-NEXT: [[ENTRY:.*]]:
+; RV64-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P]], i64 88
; RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; RV64-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; RV64-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 3)
-; RV64-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P]], i64 88
; RV64-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 132, [[UMAX]]
; RV64-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; RV64: [[VECTOR_PH]]:
-; RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; RV64-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
; RV64-NEXT: [[N_MOD_VF:%.*]] = urem i64 132, [[TMP3]]
; RV64-NEXT: [[N_VEC:%.*]] = sub i64 132, [[N_MOD_VF]]
; RV64-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 112
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll b/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
index 0a5d15a5af2f4..67df7c2aba896 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
@@ -29,8 +29,7 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP9]], 2
+; CHECK-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP8]], 2
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -89,8 +88,7 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; ZVFHMIN: [[VECTOR_PH]]:
-; ZVFHMIN-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 2
+; ZVFHMIN-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 2
; ZVFHMIN-NEXT: br label %[[VECTOR_BODY:.*]]
; ZVFHMIN: [[VECTOR_BODY]]:
; ZVFHMIN-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -171,8 +169,7 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP9]], 2
+; CHECK-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP8]], 2
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -231,8 +228,7 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; ZVFHMIN: [[VECTOR_PH]]:
-; ZVFHMIN-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 2
+; ZVFHMIN-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 2
; ZVFHMIN-NEXT: br label %[[VECTOR_BODY:.*]]
; ZVFHMIN: [[VECTOR_BODY]]:
; ZVFHMIN-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -313,8 +309,7 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP9]], 1
+; CHECK-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP8]], 1
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -373,8 +368,7 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; ZVFHMIN: [[VECTOR_PH]]:
-; ZVFHMIN-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 1
+; ZVFHMIN-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 1
; ZVFHMIN-NEXT: br label %[[VECTOR_BODY:.*]]
; ZVFHMIN: [[VECTOR_BODY]]:
; ZVFHMIN-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -455,8 +449,7 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP9]], 1
+; CHECK-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP8]], 1
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -515,8 +508,7 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; ZVFHMIN: [[VECTOR_PH]]:
-; ZVFHMIN-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 1
+; ZVFHMIN-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 1
; ZVFHMIN-NEXT: br label %[[VECTOR_BODY:.*]]
; ZVFHMIN: [[VECTOR_BODY]]:
; ZVFHMIN-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
index 022f491eb8d04..606db82f11eaf 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
@@ -20,8 +20,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
; OUTLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], [[UMAX]]
; OUTLOOP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; OUTLOOP: vector.ph:
-; OUTLOOP-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; OUTLOOP-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 2
+; OUTLOOP-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP0]], 2
; OUTLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP3]]
; OUTLOOP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; OUTLOOP-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -71,8 +70,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
; INLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], [[UMAX]]
; INLOOP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; INLOOP: vector.ph:
-; INLOOP-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; INLOOP-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 3
+; INLOOP-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP0]], 3
; INLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP3]]
; INLOOP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; INLOOP-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -201,8 +199,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
; OUTLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[UMAX]]
; OUTLOOP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; OUTLOOP: vector.ph:
-; OUTLOOP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; OUTLOOP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; OUTLOOP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; OUTLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; OUTLOOP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; OUTLOOP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
@@ -246,8 +243,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
; INLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[UMAX]]
; INLOOP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; INLOOP: vector.ph:
-; INLOOP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; INLOOP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; INLOOP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; INLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; INLOOP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; INLOOP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
index bac0b0bd01904..da32ef49ddb14 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
@@ -16,8 +16,7 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[UMAX]]
; SCALAR_EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; SCALAR_EPILOGUE: vector.ph:
-; SCALAR_EPILOGUE-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; SCALAR_EPILOGUE-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP0]], 4
; SCALAR_EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
; SCALAR_EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i32 1024, [[N_MOD_VF]]
; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
@@ -190,8 +189,7 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[UMAX]]
; SCALAR_EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; SCALAR_EPILOGUE: vector.ph:
-; SCALAR_EPILOGUE-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; SCALAR_EPILOGUE-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP0]], 4
; SCALAR_EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
; SCALAR_EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i32 1024, [[N_MOD_VF]]
; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll
index f5458c88f24a2..e2dbe7a0e41a4 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll
@@ -62,9 +62,9 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; CHECK-LABEL: define i64 @select_decreasing_induction_icmp_non_const_start(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
-; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[UMIN]]
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP1]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
@@ -77,7 +77,7 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP1]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP8]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = sub nsw i64 0, [[TMP5]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
index e80fc7a5328d2..85075addf1efa 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
@@ -19,8 +19,7 @@ define i32 @vdot4a(ptr %a, ptr %b) #0 {
; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; V-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; V: vector.ph:
-; V-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; V-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; V-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; V-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; V-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -53,8 +52,7 @@ define i32 @vdot4a(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; ZVDOT4A8I-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; ZVDOT4A8I: vector.ph:
-; ZVDOT4A8I-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; ZVDOT4A8I-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; ZVDOT4A8I-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -214,8 +212,7 @@ define i32 @vdot4au(ptr %a, ptr %b) #0 {
; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; V-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; V: vector.ph:
-; V-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; V-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; V-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; V-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; V-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -248,8 +245,7 @@ define i32 @vdot4au(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; ZVDOT4A8I-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; ZVDOT4A8I: vector.ph:
-; ZVDOT4A8I-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; ZVDOT4A8I-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; ZVDOT4A8I-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -409,8 +405,7 @@ define i32 @vdot4asu(ptr %a, ptr %b) #0 {
; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; V-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; V: vector.ph:
-; V-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; V-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; V-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; V-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; V-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -443,8 +438,7 @@ define i32 @vdot4asu(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; ZVDOT4A8I-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; ZVDOT4A8I: vector.ph:
-; ZVDOT4A8I-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; ZVDOT4A8I-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; ZVDOT4A8I-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -603,8 +597,7 @@ define i32 @vdot4asu2(ptr %a, ptr %b) #0 {
; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; V-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; V: vector.ph:
-; V-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; V-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; V-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; V-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; V-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -637,8 +630,7 @@ define i32 @vdot4asu2(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; ZVDOT4A8I-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; ZVDOT4A8I: vector.ph:
-; ZVDOT4A8I-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; ZVDOT4A8I-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; ZVDOT4A8I-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index aa2cb452ecec1..7e21b40b3f8b1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -703,8 +703,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; NOSTRIDED-UF2-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
; NOSTRIDED-UF2-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NOSTRIDED-UF2: [[VECTOR_PH]]:
-; NOSTRIDED-UF2-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-UF2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP6]], 2
+; NOSTRIDED-UF2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP0]], 2
; NOSTRIDED-UF2-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP9]], 1
; NOSTRIDED-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP7]]
; NOSTRIDED-UF2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
@@ -891,8 +890,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-UF2-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; STRIDED-UF2-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; STRIDED-UF2: [[VECTOR_PH]]:
-; STRIDED-UF2-NEXT: [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-UF2-NEXT: [[TMP29:%.*]] = shl nuw i64 [[TMP28]], 2
+; STRIDED-UF2-NEXT: [[TMP29:%.*]] = shl nuw i64 [[TMP0]], 2
; STRIDED-UF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP29]], i64 0
; STRIDED-UF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; STRIDED-UF2-NEXT: [[TMP30:%.*]] = shl nuw i64 [[TMP29]], 1
@@ -1222,8 +1220,7 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
; STRIDED-UF2-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; STRIDED-UF2-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; STRIDED-UF2: [[VECTOR_PH]]:
-; STRIDED-UF2-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-UF2-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; STRIDED-UF2-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP0]], 2
; STRIDED-UF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
; STRIDED-UF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; STRIDED-UF2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
index a9ffaf2adaf3d..a95b34560038f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
@@ -75,8 +75,7 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -194,8 +193,7 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -313,8 +311,7 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -432,8 +429,7 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -551,8 +547,7 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -670,8 +665,7 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -789,8 +783,7 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -908,8 +901,7 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1027,8 +1019,7 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1146,8 +1137,7 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1265,8 +1255,7 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1384,8 +1373,7 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1503,8 +1491,7 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1626,8 +1613,7 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP9]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1747,8 +1733,7 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP9]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1868,8 +1853,7 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP9]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1989,8 +1973,7 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP9]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -2163,8 +2146,7 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP9]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll
index 5e51a2d178861..bfad85a7f9e67 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll
@@ -90,8 +90,7 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
; NO-VP-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; NO-VP-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 2
+; NO-VP-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP12]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -232,8 +231,7 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
; NO-VP-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; NO-VP-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 2
+; NO-VP-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP12]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -374,8 +372,7 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
; NO-VP-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; NO-VP-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 2
+; NO-VP-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP12]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -516,8 +513,7 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
; NO-VP-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; NO-VP-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 2
+; NO-VP-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP12]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -645,8 +641,7 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP15]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP10]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -767,8 +762,7 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP15]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP10]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -893,8 +887,7 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP17]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP10]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1025,8 +1018,7 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP17]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP10]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1153,8 +1145,7 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP9]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
index 053d33e7238bf..40f7abcfcd9f9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
@@ -72,8 +72,7 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; NO-VP-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP9]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -189,8 +188,7 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; NO-VP-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP9]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -306,8 +304,7 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; NO-VP-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP9]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -423,8 +420,7 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; NO-VP-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -540,8 +536,7 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; NO-VP-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP7]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -661,8 +656,7 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP9]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -782,8 +776,7 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP9]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -903,8 +896,7 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP9]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1024,8 +1016,7 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP9]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1145,8 +1136,7 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP15]]
; NO-VP-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 1
+; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP10]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1236,8 +1226,7 @@ define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP13]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index a5a1c709251ec..dd61e30a31725 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -60,8 +60,7 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll
index f6c3ebeb5c585..0fd31aa0bf518 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll
@@ -44,8 +44,7 @@ define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP4]], 1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP11]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -139,8 +138,7 @@ define void @test_sdiv_divisor_invariant_nonconst(ptr noalias %a, i64 %b, ptr no
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; NO-VP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[B]], i64 0
@@ -233,8 +231,7 @@ define void @test_sdiv_both_invariant_nonconst(ptr noalias %a, i64 %b, i64 %b2,
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; NO-VP-NEXT: [[TMP4:%.*]] = sdiv i64 [[B]], [[B2]]
@@ -325,8 +322,7 @@ define void @test_sdiv_divisor_invariant_minusone(ptr noalias %a, ptr noalias %c
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -412,8 +408,7 @@ define void @test_sdiv_divisor_invariant_safeimm(ptr noalias %a, ptr noalias %c)
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -501,8 +496,7 @@ define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP4]], 1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP11]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -596,8 +590,7 @@ define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP4]], 1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP11]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -691,8 +684,7 @@ define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
-; NO-VP-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP4]], 1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP11]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
index 2e0a77ff57f87..60be412afe18f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
@@ -153,8 +153,7 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -269,8 +268,7 @@ define void @store_factor_4_with_gap(i32 %n, ptr noalias %a) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT: [[TMP9:%.*]] = shl nuw i32 [[TMP2]], 2
+; NO-VP-NEXT: [[TMP9:%.*]] = shl nuw i32 [[TMP0]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP9]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
@@ -554,10 +552,7 @@ exit:
define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; IF-EVL-LABEL: @load_factor_4_reverse(
; IF-EVL-NEXT: entry:
-; IF-EVL-NEXT: [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1
-; IF-EVL-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)
-; IF-EVL-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
-; IF-EVL-NEXT: [[TMP2:%.*]] = shl i64 [[N]], 4
+; IF-EVL-NEXT: [[TMP2:%.*]] = shl i64 [[N:%.*]], 4
; IF-EVL-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 12
; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP3]]
; IF-EVL-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP2]], 8
@@ -565,6 +560,9 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; IF-EVL-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP2]], 4
; IF-EVL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; IF-EVL-NEXT: [[TMP20:%.*]] = add nsw i64 [[N]], -1
+; IF-EVL-NEXT: [[TMP21:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP20]], i64 0)
+; IF-EVL-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[TMP21]]
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
; IF-EVL: vector.ph:
; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -600,13 +598,7 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
;
; NO-VP-LABEL: @load_factor_4_reverse(
; NO-VP-NEXT: entry:
-; NO-VP-NEXT: [[TMP27:%.*]] = add nsw i64 [[N:%.*]], -1
-; NO-VP-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP27]], i64 0)
-; NO-VP-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP3]], i64 8)
-; NO-VP-NEXT: [[TMP8:%.*]] = shl i64 [[N]], 4
+; NO-VP-NEXT: [[TMP8:%.*]] = shl i64 [[N:%.*]], 4
; NO-VP-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP8]], 12
; NO-VP-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP10]]
; NO-VP-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP8]], 8
@@ -614,10 +606,15 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; NO-VP-NEXT: [[TMP9:%.*]] = add nuw nsw i64 [[TMP8]], 4
; NO-VP-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP9]]
; NO-VP-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP8]]
+; NO-VP-NEXT: [[TMP27:%.*]] = add nsw i64 [[N]], -1
+; NO-VP-NEXT: [[TMP28:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP27]], i64 0)
+; NO-VP-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[TMP28]]
+; NO-VP-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT: [[TMP29:%.*]] = shl nuw i64 [[TMP4]], 2
+; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP29]], i64 8)
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
-; NO-VP-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv32.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv32.ll
index 166ee488d05a4..09025ccceb4bb 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv32.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv32.ll
@@ -38,8 +38,7 @@ define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT: [[TMP11:%.*]] = shl nuw i32 [[TMP2]], 2
+; NO-VP-NEXT: [[TMP11:%.*]] = shl nuw i32 [[TMP0]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP11]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-masked-loadstore.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-masked-loadstore.ll
index e58e154e5be84..2e5961b3d0d5f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-masked-loadstore.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-masked-loadstore.ll
@@ -42,8 +42,7 @@ define void @masked_loadstore(ptr noalias %a, ptr noalias %b, i64 %n) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP11]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; NO-VP-NEXT: [[INC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label [[FOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
index 2e59792719dd1..c4ebc4f2ad9c2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
@@ -45,8 +45,7 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[STARTVAL:%.*]], [[N_VEC]]
@@ -150,8 +149,7 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[ENTRY:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[STARTVAL1:%.*]], [[N_VEC]]
@@ -280,8 +278,7 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP5]], 4
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 1024, [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll
index 396ed31397841..f545792c2510a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll
@@ -43,8 +43,7 @@ define void @test(ptr %p) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
; NO-VP: vector.ph:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]]
; NO-VP-NEXT: br label [[LOOP:%.*]]
@@ -300,8 +299,7 @@ define void @trivial_due_max_vscale(ptr %p) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
; NO-VP: vector.ph:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]]
; NO-VP-NEXT: br label [[LOOP:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
index 2c9dfec8c1910..1f7571ba3d286 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
@@ -42,8 +42,7 @@ define void @load_store_interleave_group(ptr noalias %data) {
; EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[UMAX]]
; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; EPILOGUE: [[VECTOR_PH]]:
-; EPILOGUE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP2]]
+; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP0]]
; EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
; EPILOGUE: [[VECTOR_BODY]]:
@@ -52,7 +51,7 @@ define void @load_store_interleave_group(ptr noalias %data) {
; EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP4]]
; EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP5]], align 8
; EPILOGUE-NEXT: store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 8
-; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
; EPILOGUE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; EPILOGUE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; EPILOGUE: [[MIDDLE_BLOCK]]:
@@ -102,10 +101,10 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; CHECK-LABEL: define void @interleave_group_with_countable_early_exit(
; CHECK-SAME: i64 [[N:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 38)
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], [[UMAX]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
@@ -125,17 +124,16 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP10]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP12]], i64 [[TMP10]], i64 [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP12]], i64 [[TMP1]], i64 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP13]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr { i64, i64 }, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <vscale x 2 x i64> zeroinitializer, ptr [[TMP14]], align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -163,10 +161,10 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; EPILOGUE-LABEL: define void @interleave_group_with_countable_early_exit(
; EPILOGUE-SAME: i64 [[N:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {
; EPILOGUE-NEXT: [[ENTRY:.*]]:
+; EPILOGUE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; EPILOGUE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; EPILOGUE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; EPILOGUE-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 40)
-; EPILOGUE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], [[UMAX]]
; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; EPILOGUE: [[VECTOR_SCEVCHECK]]:
@@ -186,17 +184,16 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; EPILOGUE-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
; EPILOGUE-NEXT: br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; EPILOGUE: [[VECTOR_PH]]:
-; EPILOGUE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP10]]
+; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP1]]
; EPILOGUE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; EPILOGUE-NEXT: [[TMP13:%.*]] = select i1 [[TMP12]], i64 [[TMP10]], i64 [[N_MOD_VF]]
+; EPILOGUE-NEXT: [[TMP13:%.*]] = select i1 [[TMP12]], i64 [[TMP1]], i64 [[N_MOD_VF]]
; EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP13]]
; EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
; EPILOGUE: [[VECTOR_BODY]]:
; EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; EPILOGUE-NEXT: [[TMP14:%.*]] = getelementptr { i64, i64 }, ptr [[DST]], i64 [[INDEX]]
; EPILOGUE-NEXT: store <vscale x 2 x i64> zeroinitializer, ptr [[TMP14]], align 8
-; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; EPILOGUE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; EPILOGUE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; EPILOGUE: [[MIDDLE_BLOCK]]:
@@ -287,8 +284,7 @@ define void @load_store_interleave_group_i32(ptr noalias %data) {
; EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[UMAX]]
; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; EPILOGUE: [[VECTOR_PH]]:
-; EPILOGUE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP2]]
+; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP0]]
; EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
; EPILOGUE: [[VECTOR_BODY]]:
@@ -297,7 +293,7 @@ define void @load_store_interleave_group_i32(ptr noalias %data) {
; EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[TMP4]]
; EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 8
; EPILOGUE-NEXT: store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP5]], align 8
-; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
; EPILOGUE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; EPILOGUE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; EPILOGUE: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
index b3b83e80ae17c..6787cb8997939 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
@@ -42,8 +42,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[UMAX]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll b/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
index a43bb14392061..d19c09486a95b 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
@@ -204,11 +204,10 @@ exit:
define void @scev_smax_expanded(ptr %dst, i64 %n) {
; CHECK-LABEL: VPlan for loop in 'scev_smax_expanded'
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
-; CHECK-NEXT: Live-in ir<%smax> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: IR %smax = call i64 @llvm.smax.i64(i64 %n, i64 1)
-; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%smax>, ir<4>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = call i64 @llvm.smax(ir<%n>, ir<1>)
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult vp<[[VP2]]>, ir<4>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
;
@@ -231,13 +230,12 @@ exit:
define void @scev_smin_expanded(ptr %dst, i64 %n) {
; CHECK-LABEL: VPlan for loop in 'scev_smin_expanded'
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
-; CHECK-NEXT: Live-in ir<%1> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: IR %0 = add nsw i64 %n, -1
-; CHECK-NEXT: IR %smin = call i64 @llvm.smin.i64(i64 %0, i64 0)
-; CHECK-NEXT: IR %1 = sub i64 %n, %smin
-; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%1>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = add nsw ir<%n>, ir<-1>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP3:%[0-9]+]]> = call i64 @llvm.smin(vp<[[VP2]]>, ir<0>)
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = sub ir<%n>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult vp<[[VP4]]>, ir<4>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll b/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
index bf07ee4f6f21e..ae14f0b233c6e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
@@ -35,9 +35,9 @@ define i32 @main(ptr %ptr) {
; CHECK-NEXT: [[DOTPROMOTED:%.*]] = load i32, ptr @a, align 16
; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[CONV3]], -1
; CHECK-NEXT: [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 1
; CHECK-NEXT: [[UMIN1:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP0]], i32 [[TMP2]])
-; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], [[UMIN1]]
+; CHECK-NEXT: [[TMP11:%.*]] = sub i32 [[TMP2]], [[UMIN1]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP11]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP4]], 24
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
; CHECK: vector.scevcheck:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
index 8412b617ad44c..ddb1905f3ffc4 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
@@ -704,9 +704,9 @@ define void @loaded_address_used_by_load_through_blend(i64 %start, ptr noalias %
; I32-LABEL: define void @loaded_address_used_by_load_through_blend(
; I32-SAME: i64 [[START:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[SRC_2:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; I32-NEXT: [[ENTRY:.*:]]
-; I32-NEXT: [[TMP0:%.*]] = add i64 [[START]], 1
; I32-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[START]], i64 100)
-; I32-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[SMIN]]
+; I32-NEXT: [[TMP92:%.*]] = sub i64 [[START]], [[SMIN]]
+; I32-NEXT: [[TMP1:%.*]] = add i64 [[TMP92]], 1
; I32-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 8
; I32-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; I32: [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/cast-induction.ll b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
index 6589b0dc2910c..b087ed0e0b412 100644
--- a/llvm/test/Transforms/LoopVectorize/cast-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
@@ -420,12 +420,14 @@ define i64 @induction_cast_chain_cleared_by_dce(i64 %n, i64 %mask.init) {
; VF4-SAME: i64 [[N:%.*]], i64 [[MASK_INIT:%.*]]) {
; VF4-NEXT: [[ENTRY:.*]]:
; VF4-NEXT: [[MASK:%.*]] = and i64 [[MASK_INIT]], -4294967296
+; VF4-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[MASK]], 1
; VF4-NEXT: [[TMP0:%.*]] = add nuw nsw i64 [[MASK]], 2
-; VF4-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 [[TMP0]])
+; VF4-NEXT: [[TMP2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 [[TMP0]])
+; VF4-NEXT: [[TMP21:%.*]] = lshr i64 [[MASK_INIT]], 32
+; VF4-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], -4294967296
+; VF4-NEXT: [[SMAX1:%.*]] = add i64 [[TMP2]], [[TMP22]]
; VF4-NEXT: [[TMP1:%.*]] = add i64 [[SMAX1]], -1
-; VF4-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[MASK]]
-; VF4-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[MASK]], 1
-; VF4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; VF4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
; VF4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; VF4: [[VECTOR_SCEVCHECK]]:
; VF4-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[MASK]], 2
@@ -442,8 +444,8 @@ define i64 @induction_cast_chain_cleared_by_dce(i64 %n, i64 %mask.init) {
; VF4-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[IDENT_CHECK]]
; VF4-NEXT: br i1 [[TMP13]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; VF4: [[VECTOR_PH]]:
-; VF4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
-; VF4-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; VF4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; VF4-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; VF4-NEXT: [[TMP14:%.*]] = mul i64 [[N_VEC]], [[TMP3]]
; VF4-NEXT: [[TMP15:%.*]] = add i64 1, [[TMP14]]
; VF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
@@ -465,7 +467,7 @@ define i64 @induction_cast_chain_cleared_by_dce(i64 %n, i64 %mask.init) {
; VF4-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; VF4: [[MIDDLE_BLOCK]]:
; VF4-NEXT: [[TMP20:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP18]])
-; VF4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; VF4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; VF4-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VF4: [[SCALAR_PH]]:
; VF4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ], [ 1, %[[VECTOR_SCEVCHECK]] ]
@@ -488,12 +490,14 @@ define i64 @induction_cast_chain_cleared_by_dce(i64 %n, i64 %mask.init) {
; IC2-SAME: i64 [[N:%.*]], i64 [[MASK_INIT:%.*]]) {
; IC2-NEXT: [[ENTRY:.*]]:
; IC2-NEXT: [[MASK:%.*]] = and i64 [[MASK_INIT]], -4294967296
+; IC2-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[MASK]], 1
; IC2-NEXT: [[TMP0:%.*]] = add nuw nsw i64 [[MASK]], 2
-; IC2-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 [[TMP0]])
+; IC2-NEXT: [[TMP2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 [[TMP0]])
+; IC2-NEXT: [[TMP23:%.*]] = lshr i64 [[MASK_INIT]], 32
+; IC2-NEXT: [[TMP24:%.*]] = mul i64 [[TMP23]], -4294967296
+; IC2-NEXT: [[SMAX1:%.*]] = add i64 [[TMP2]], [[TMP24]]
; IC2-NEXT: [[TMP1:%.*]] = add i64 [[SMAX1]], -1
-; IC2-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[MASK]]
-; IC2-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[MASK]], 1
-; IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2
+; IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 2
; IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; IC2: [[VECTOR_SCEVCHECK]]:
; IC2-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[MASK]], 2
@@ -510,8 +514,8 @@ define i64 @induction_cast_chain_cleared_by_dce(i64 %n, i64 %mask.init) {
; IC2-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[IDENT_CHECK]]
; IC2-NEXT: br i1 [[TMP13]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; IC2: [[VECTOR_PH]]:
-; IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2
-; IC2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 2
+; IC2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; IC2-NEXT: [[TMP14:%.*]] = mul i64 [[N_VEC]], [[TMP3]]
; IC2-NEXT: [[TMP15:%.*]] = add i64 1, [[TMP14]]
; IC2-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -530,7 +534,7 @@ define i64 @induction_cast_chain_cleared_by_dce(i64 %n, i64 %mask.init) {
; IC2-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; IC2: [[MIDDLE_BLOCK]]:
; IC2-NEXT: [[BIN_RDX:%.*]] = add i64 [[TMP21]], [[TMP20]]
-; IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; IC2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; IC2: [[SCALAR_PH]]:
; IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ], [ 1, %[[VECTOR_SCEVCHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
index 264ae9d1b3471..978083c26250f 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
@@ -504,9 +504,9 @@ define i64 @findlast_expr_flipped_select_anyof(ptr %a, ptr %b, i64 %rdx.start, i
; CHECK-LABEL: define i64 @findlast_expr_flipped_select_anyof(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
-; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
+; CHECK-NEXT: [[TMP18:%.*]] = sub i64 [[N]], [[UMIN]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP18]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP1]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
index 6f00a52febd53..dd22f274f2d0a 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
@@ -700,9 +700,9 @@ define i64 @findlast_expr_flipped_select_anyof(ptr %a, ptr %b, i64 %rdx.start, i
; CHECK-LABEL: define i64 @findlast_expr_flipped_select_anyof(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
-; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
+; CHECK-NEXT: [[TMP18:%.*]] = sub i64 [[N]], [[UMIN]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP18]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/find-last.ll b/llvm/test/Transforms/LoopVectorize/find-last.ll
index 9fd06ab4ae6b4..4df5d7f93a0bd 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last.ll
@@ -204,13 +204,13 @@ define i32 @loop_inv_select_condition_issue_182152(i32 %iv.start, i32 %a, i32 %b
; CHECK-NEXT: entry:
; CHECK-NEXT: [[INV_COND:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[IV_START:%.*]], i32 1)
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[SMAX]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[IV_START]]
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4
+; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[SMAX]], [[IV_START]]
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP11]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP11]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP11]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[INV_COND]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[IV_START]], [[N_VEC]]
@@ -234,7 +234,7 @@ define i32 @loop_inv_select_condition_issue_182152(i32 %iv.start, i32 %a, i32 %b
; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP8]], <4 x i1> [[TMP7]], i32 0)
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP11]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP2]], [[MIDDLE_BLOCK]] ], [ [[IV_START]], [[ENTRY:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index 241c4bb4de3aa..ea2a19d3c3125 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -2598,9 +2598,9 @@ define i32 @sink_into_replication_region(i32 %y) {
; UNROLL-NO-IC-LABEL: define i32 @sink_into_replication_region(
; UNROLL-NO-IC-SAME: i32 [[Y:%.*]]) {
; UNROLL-NO-IC-NEXT: [[BB:.*:]]
-; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[Y]], 1
; UNROLL-NO-IC-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1)
-; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = sub i32 [[Y]], [[SMIN]]
+; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i32 [[TMP5]], 1
; UNROLL-NO-IC-NEXT: br label %[[VECTOR_PH:.*]]
; UNROLL-NO-IC: [[VECTOR_PH]]:
; UNROLL-NO-IC-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 7
@@ -2711,9 +2711,9 @@ define i32 @sink_into_replication_region(i32 %y) {
; UNROLL-NO-VF-LABEL: define i32 @sink_into_replication_region(
; UNROLL-NO-VF-SAME: i32 [[Y:%.*]]) {
; UNROLL-NO-VF-NEXT: [[BB:.*:]]
-; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i32 [[Y]], 1
; UNROLL-NO-VF-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1)
-; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = sub i32 [[Y]], [[SMIN]]
+; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i32 [[TMP4]], 1
; UNROLL-NO-VF-NEXT: br label %[[VECTOR_PH:.*]]
; UNROLL-NO-VF: [[VECTOR_PH]]:
; UNROLL-NO-VF-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1
@@ -2760,9 +2760,9 @@ define i32 @sink_into_replication_region(i32 %y) {
; SINK-AFTER-LABEL: define i32 @sink_into_replication_region(
; SINK-AFTER-SAME: i32 [[Y:%.*]]) {
; SINK-AFTER-NEXT: [[BB:.*:]]
-; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[Y]], 1
; SINK-AFTER-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1)
-; SINK-AFTER-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; SINK-AFTER-NEXT: [[TMP4:%.*]] = sub i32 [[Y]], [[SMIN]]
+; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i32 [[TMP4]], 1
; SINK-AFTER-NEXT: br label %[[VECTOR_PH:.*]]
; SINK-AFTER: [[VECTOR_PH]]:
; SINK-AFTER-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3
@@ -2851,9 +2851,9 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
; UNROLL-NO-IC-LABEL: define i32 @sink_into_replication_region_multiple(
; UNROLL-NO-IC-SAME: ptr [[X:%.*]], i32 [[Y:%.*]]) {
; UNROLL-NO-IC-NEXT: [[BB:.*:]]
-; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[Y]], 1
; UNROLL-NO-IC-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1)
-; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = sub i32 [[Y]], [[SMIN]]
+; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i32 [[TMP2]], 1
; UNROLL-NO-IC-NEXT: br label %[[VECTOR_PH:.*]]
; UNROLL-NO-IC: [[VECTOR_PH]]:
; UNROLL-NO-IC-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 7
@@ -3020,9 +3020,9 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
; UNROLL-NO-VF-LABEL: define i32 @sink_into_replication_region_multiple(
; UNROLL-NO-VF-SAME: ptr [[X:%.*]], i32 [[Y:%.*]]) {
; UNROLL-NO-VF-NEXT: [[BB:.*:]]
-; UNROLL-NO-VF-NEXT: [[TMP0:%.*]] = add i32 [[Y]], 1
; UNROLL-NO-VF-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1)
-; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = sub i32 [[Y]], [[SMIN]]
+; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = add i32 [[TMP2]], 1
; UNROLL-NO-VF-NEXT: br label %[[VECTOR_PH:.*]]
; UNROLL-NO-VF: [[VECTOR_PH]]:
; UNROLL-NO-VF-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1
@@ -3081,9 +3081,9 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
; SINK-AFTER-LABEL: define i32 @sink_into_replication_region_multiple(
; SINK-AFTER-SAME: ptr [[X:%.*]], i32 [[Y:%.*]]) {
; SINK-AFTER-NEXT: [[BB:.*:]]
-; SINK-AFTER-NEXT: [[TMP0:%.*]] = add i32 [[Y]], 1
; SINK-AFTER-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[Y]], i32 1)
-; SINK-AFTER-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; SINK-AFTER-NEXT: [[TMP2:%.*]] = sub i32 [[Y]], [[SMIN]]
+; SINK-AFTER-NEXT: [[TMP1:%.*]] = add i32 [[TMP2]], 1
; SINK-AFTER-NEXT: br label %[[VECTOR_PH:.*]]
; SINK-AFTER: [[VECTOR_PH]]:
; SINK-AFTER-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/induction-ptrcasts.ll b/llvm/test/Transforms/LoopVectorize/induction-ptrcasts.ll
index ff7c8e8fc2255..a744025cf3889 100644
--- a/llvm/test/Transforms/LoopVectorize/induction-ptrcasts.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction-ptrcasts.ll
@@ -9,7 +9,8 @@ define void @int_iv_based_on_pointer_iv(ptr %A) {
; VF1-SAME: ptr [[A:%.*]]) {
; VF1-NEXT: [[ENTRY:.*:]]
; VF1-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 add (i64 ptrtoint (ptr @f to i64), i64 -4), i64 0)
-; VF1-NEXT: [[TMP0:%.*]] = sub i64 add (i64 ptrtoint (ptr @f to i64), i64 -1), [[SMIN]]
+; VF1-NEXT: [[TMP6:%.*]] = sub i64 ptrtoint (ptr @f to i64), [[SMIN]]
+; VF1-NEXT: [[TMP0:%.*]] = add i64 [[TMP6]], -1
; VF1-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 2
; VF1-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; VF1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2
@@ -40,7 +41,8 @@ define void @int_iv_based_on_pointer_iv(ptr %A) {
; VF2-SAME: ptr [[A:%.*]]) {
; VF2-NEXT: [[ENTRY:.*:]]
; VF2-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 add (i64 ptrtoint (ptr @f to i64), i64 -4), i64 0)
-; VF2-NEXT: [[TMP0:%.*]] = sub i64 add (i64 ptrtoint (ptr @f to i64), i64 -1), [[SMIN]]
+; VF2-NEXT: [[TMP7:%.*]] = sub i64 ptrtoint (ptr @f to i64), [[SMIN]]
+; VF2-NEXT: [[TMP0:%.*]] = add i64 [[TMP7]], -1
; VF2-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 2
; VF2-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; VF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/invalidate-scev-at-scope-after-vectorization.ll b/llvm/test/Transforms/LoopVectorize/invalidate-scev-at-scope-after-vectorization.ll
index 32de44ce8aac1..357a2c4f2444d 100644
--- a/llvm/test/Transforms/LoopVectorize/invalidate-scev-at-scope-after-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/invalidate-scev-at-scope-after-vectorization.ll
@@ -34,9 +34,9 @@ define void @test_invalidate_scevs_at_scope(ptr %p) {
; CHECK-NEXT: [[ADD_LCSSA1:%.*]] = phi i32 [ [[ADD_1]], %[[LOOP_1]] ]
; CHECK-NEXT: [[ADD_LCSSA:%.*]] = add i32 [[TMP4]], 100
; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[ADD_LCSSA]], i32 100)
-; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[SMAX]], -100
-; CHECK-NEXT: [[TMP5:%.*]] = sub i32 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = sub i32 [[SMAX]], [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP5]], -100
+; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP8]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP6]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP7]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH2:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
index 69fc31a3ee23a..930c78529e72f 100644
--- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
@@ -201,22 +201,22 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
-; CHECK: pred.store.if5:
+; CHECK: pred.store.if4:
; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META19]]
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE6]]
-; CHECK: pred.store.continue6:
+; CHECK: pred.store.continue5:
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
-; CHECK: pred.store.if7:
+; CHECK: pred.store.if6:
; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META19]]
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE8]]
-; CHECK: pred.store.continue8:
+; CHECK: pred.store.continue7:
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10]]
-; CHECK: pred.store.if9:
+; CHECK: pred.store.if8:
; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META19]]
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE10]]
-; CHECK: pred.store.continue10:
+; CHECK: pred.store.continue9:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
index 65714e9f30595..558ea6d885611 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
@@ -1105,9 +1105,9 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; IC1VF4-LABEL: define i64 @select_decreasing_induction_icmp_non_const_start(
; IC1VF4-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
; IC1VF4-NEXT: [[ENTRY:.*]]:
-; IC1VF4-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; IC1VF4-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
-; IC1VF4-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
+; IC1VF4-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[UMIN]]
+; IC1VF4-NEXT: [[TMP1:%.*]] = add i64 [[TMP6]], 1
; IC1VF4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
; IC1VF4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; IC1VF4: [[VECTOR_PH]]:
@@ -1171,9 +1171,9 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; IC4VF4-LABEL: define i64 @select_decreasing_induction_icmp_non_const_start(
; IC4VF4-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
; IC4VF4-NEXT: [[ENTRY:.*]]:
-; IC4VF4-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; IC4VF4-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
-; IC4VF4-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
+; IC4VF4-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[UMIN]]
+; IC4VF4-NEXT: [[TMP1:%.*]] = add i64 [[TMP6]], 1
; IC4VF4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 16
; IC4VF4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; IC4VF4: [[VECTOR_PH]]:
@@ -1279,9 +1279,9 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; IC4VF1-LABEL: define i64 @select_decreasing_induction_icmp_non_const_start(
; IC4VF1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
; IC4VF1-NEXT: [[ENTRY:.*]]:
-; IC4VF1-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; IC4VF1-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
-; IC4VF1-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
+; IC4VF1-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[UMIN]]
+; IC4VF1-NEXT: [[TMP1:%.*]] = add i64 [[TMP6]], 1
; IC4VF1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
; IC4VF1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[EXIT:.*]], label %[[VECTOR_PH:.*]]
; IC4VF1: [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
index cc1d4b2064565..d94ac67ff81a5 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
@@ -64,9 +64,9 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; CHECK-LABEL: define i64 @select_decreasing_induction_icmp_non_const_start(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
-; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
+; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[N]], [[UMIN]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP4]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP1]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index efe9ed64b203f..94078c8413f41 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -322,10 +322,10 @@ define void @PR30742(ptr %p) {
; CHECK-NEXT: [[TMP02:%.*]] = icmp slt i32 [[TMP01]], 1
; CHECK-NEXT: [[TMP03:%.*]] = select i1 [[TMP02]], i32 1, i32 [[TMP01]]
; CHECK-NEXT: [[TMP04:%.*]] = add nsw i32 [[TMP03]], -7
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[TMP03]], -8
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[TMP03]], -15
; CHECK-NEXT: [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP1]], i32 0)
-; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[TMP0]], [[SMIN1]]
+; CHECK-NEXT: [[TMP9:%.*]] = sub i32 [[TMP03]], [[SMIN1]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP9]], -8
; CHECK-NEXT: [[TMP3:%.*]] = lshr i32 [[TMP2]], 3
; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK4:%.*]] = icmp ult i32 [[TMP4]], 2
@@ -359,10 +359,10 @@ define void @PR30742(ptr %p) {
; CHECK-NEXT: [[TMP10:%.*]] = icmp slt i32 [[TMP09]], 1
; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i32 1, i32 [[TMP09]]
; CHECK-NEXT: [[TMP12:%.*]] = add nsw i32 [[TMP11]], -7
-; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[TMP11]], -8
; CHECK-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP11]], -15
; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP8]], i32 0)
-; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[TMP7]], [[SMIN]]
+; CHECK-NEXT: [[TMP19:%.*]] = sub i32 [[TMP11]], [[SMIN]]
+; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP19]], -8
; CHECK-NEXT: [[TMP11:%.*]] = lshr i32 [[TMP10]], 3
; CHECK-NEXT: [[TMP14:%.*]] = add nuw nsw i32 [[TMP11]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP14]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
index bc0a947280cb5..8e233730cd976 100644
--- a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
@@ -198,7 +198,7 @@ define i32 @loop_requires_scev_predicate(ptr %dest, i32 %end) {
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i1> [[TMP11]], i64 1
; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
-; CHECK: pred.store.if3:
+; CHECK: pred.store.if2:
; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[DEST]], i64 [[TMP20]]
; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
@@ -206,7 +206,7 @@ define i32 @loop_requires_scev_predicate(ptr %dest, i32 %end) {
; CHECK-NEXT: [[TMP24:%.*]] = add i32 [[TMP22]], [[TMP23]]
; CHECK-NEXT: store i32 [[TMP24]], ptr [[TMP21]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE4]]
-; CHECK: pred.store.continue4:
+; CHECK: pred.store.continue3:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
index 7dc1228346873..491882b796ef1 100644
--- a/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
+++ b/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
@@ -80,13 +80,14 @@ define void @pr49900(i32 %x, ptr %ptr) {
; CHECK-NEXT: [[IV_1:%.*]] = phi i32 [ 0, %[[LOOP_1_PH]] ], [ [[IV_3_NEXT_LCSSA:%.*]], %[[LOOP_1_LATCH:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[IV_1]], 12
; CHECK-NEXT: [[SMAX2:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP0]], i32 65537)
-; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SMAX2]], -12
-; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], [[IV_1]]
+; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[SMAX2]], [[IV_1]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP3]], -12
; CHECK-NEXT: [[UMIN3:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP2]], i32 1)
-; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[UMIN3]], 1
-; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[TMP2]], [[UMIN3]]
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[TMP3]], [[UMIN3]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP10]], -12
; CHECK-NEXT: [[TMP5:%.*]] = udiv i32 [[TMP4]], 13
-; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[UMIN3]], [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[TMP12]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK5:%.*]] = icmp ult i32 [[TMP6]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK5]], label %[[SCALAR_PH4:.*]], label %[[VECTOR_PH6:.*]]
; CHECK: [[VECTOR_PH6]]:
@@ -104,10 +105,10 @@ define void @pr49900(i32 %x, ptr %ptr) {
; CHECK-NEXT: [[CMP_N13:%.*]] = icmp eq i32 [[TMP6]], [[N_VEC8]]
; CHECK-NEXT: br i1 [[CMP_N13]], label %[[LOOP_3_PH:.*]], label %[[SCALAR_PH4]]
; CHECK: [[SCALAR_PH4]]:
-; CHECK-NEXT: [[BC_RESUME_VAL14:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK12]] ], [ [[IV_1]], %[[LOOP_1]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL11:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK12]] ], [ [[IV_1]], %[[LOOP_1]] ]
; CHECK-NEXT: br label %[[LOOP_2:.*]]
; CHECK: [[LOOP_2]]:
-; CHECK-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL14]], %[[SCALAR_PH4]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP_2]] ]
+; CHECK-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL11]], %[[SCALAR_PH4]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP_2]] ]
; CHECK-NEXT: [[TMP54:%.*]] = add i32 [[IV_2]], 12
; CHECK-NEXT: [[IV_2_NEXT]] = add i32 [[IV_2]], 13
; CHECK-NEXT: [[EXT:%.*]] = zext i32 [[IV_2_NEXT]] to i64
@@ -116,27 +117,30 @@ define void @pr49900(i32 %x, ptr %ptr) {
; CHECK-NEXT: br i1 [[C6]], label %[[LOOP_2]], label %[[LOOP_3_PH]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[LOOP_3_PH]]:
; CHECK-NEXT: [[IV_2_NEXT_LCSSA:%.*]] = phi i32 [ [[IV_2_NEXT]], %[[LOOP_2]] ], [ [[TMP8]], %[[MIDDLE_BLOCK12]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[IV_1]], 26
; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[IV_1]], 12
; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP11]], i32 65537)
-; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[SMAX]], -12
-; CHECK-NEXT: [[TMP13:%.*]] = sub i32 [[TMP12]], [[IV_1]]
+; CHECK-NEXT: [[TMP19:%.*]] = sub i32 [[SMAX]], [[IV_1]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP19]], -12
; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP13]], i32 1)
-; CHECK-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], [[UMIN]]
+; CHECK-NEXT: [[TMP22:%.*]] = sub i32 [[TMP19]], [[UMIN]]
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP22]], -12
; CHECK-NEXT: [[TMP15:%.*]] = udiv i32 [[TMP14]], 13
; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[UMIN]], [[TMP15]]
; CHECK-NEXT: [[TMP17:%.*]] = mul i32 [[TMP16]], 13
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP10]], [[TMP17]]
+; CHECK-NEXT: [[TMP30:%.*]] = add i32 [[IV_1]], [[TMP17]]
+; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP30]], 26
; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP18]], i32 65536)
-; CHECK-NEXT: [[TMP19:%.*]] = add i32 [[UMAX]], -26
-; CHECK-NEXT: [[TMP20:%.*]] = sub i32 [[TMP19]], [[IV_1]]
-; CHECK-NEXT: [[TMP21:%.*]] = sub i32 [[TMP20]], [[TMP17]]
+; CHECK-NEXT: [[TMP31:%.*]] = mul i32 [[TMP16]], -13
+; CHECK-NEXT: [[TMP20:%.*]] = sub i32 [[UMAX]], [[IV_1]]
+; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[TMP20]], [[TMP31]]
+; CHECK-NEXT: [[TMP21:%.*]] = add i32 [[TMP33]], -26
; CHECK-NEXT: [[UMIN1:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP21]], i32 1)
-; CHECK-NEXT: [[TMP22:%.*]] = add i32 [[UMIN1]], 1
; CHECK-NEXT: [[TMP23:%.*]] = sub i32 [[TMP20]], [[UMIN1]]
-; CHECK-NEXT: [[TMP24:%.*]] = sub i32 [[TMP23]], [[TMP17]]
+; CHECK-NEXT: [[TMP32:%.*]] = add i32 [[TMP23]], [[TMP31]]
+; CHECK-NEXT: [[TMP24:%.*]] = add i32 [[TMP32]], -26
; CHECK-NEXT: [[TMP25:%.*]] = udiv i32 [[TMP24]], 13
-; CHECK-NEXT: [[TMP26:%.*]] = add i32 [[TMP22]], [[TMP25]]
+; CHECK-NEXT: [[TMP35:%.*]] = add i32 [[UMIN1]], [[TMP25]]
+; CHECK-NEXT: [[TMP26:%.*]] = add i32 [[TMP35]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP26]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/pr125278.ll b/llvm/test/Transforms/LoopVectorize/pr125278.ll
index bb29e0c421847..909838157ee4a 100644
--- a/llvm/test/Transforms/LoopVectorize/pr125278.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr125278.ll
@@ -6,11 +6,11 @@ define void @pr125278(ptr %dst, i64 %n) {
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TRUE_EXT:%.*]] = zext i1 true to i32
-; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: br label %[[COND:.*]]
; CHECK: [[COND_LOOPEXIT:.*]]:
; CHECK-NEXT: br label %[[COND]]
; CHECK: [[COND]]:
+; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/pr37248.ll b/llvm/test/Transforms/LoopVectorize/pr37248.ll
index c2e26c5d66877..38093ecb1f4df 100644
--- a/llvm/test/Transforms/LoopVectorize/pr37248.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr37248.ll
@@ -16,9 +16,9 @@ define void @f1(ptr noalias %b, i1 %c, i32 %start) {
; CHECK-LABEL: define void @f1(
; CHECK-SAME: ptr noalias [[B:%.*]], i1 [[C:%.*]], i32 [[START:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[START]], 1
; CHECK-NEXT: [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[START]], i32 1)
-; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN1]]
+; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[START]], [[SMIN1]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
@@ -89,9 +89,9 @@ define void @f2(ptr noalias %b, i1 %c, i32 %start) {
; CHECK-LABEL: define void @f2(
; CHECK-SAME: ptr noalias [[B:%.*]], i1 [[C:%.*]], i32 [[START:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[START]], 1
; CHECK-NEXT: [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[START]], i32 1)
-; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN1]]
+; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[START]], [[SMIN1]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/pr38697.ll b/llvm/test/Transforms/LoopVectorize/pr38697.ll
index 6c5d11b8ab8cb..5570a1d8f7bd0 100644
--- a/llvm/test/Transforms/LoopVectorize/pr38697.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr38697.ll
@@ -38,7 +38,7 @@ define void @testCountIncrLoop(ptr %ptr, i32 %lim, i32 %count, i32 %val) mustpro
; CHECK: loop1.body:
; CHECK: while.cond.preheader:
; CHECK: while.body.preheader:
-; CHECK-NEXT: [[TMP1:%.*]] = udiv i32 [[TMP0:%.*]], [[COUNT:%.*]]
+; CHECK: [[TMP1:%.*]] = udiv i32 [[TMP0:%.*]], [[COUNT:%.*]]
; CHECK: vector.ph:
; CHECK: exit:
; CHECK: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
index f3c72a8f9e9a9..1487b891c7c0b 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
@@ -172,13 +172,13 @@ define i32 @PR35734(i32 %x, i32 %y) {
; CHECK-LABEL: @PR35734(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[X:%.*]], i32 78)
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[SMAX]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[X]]
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4
+; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[SMAX]], [[X]]
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP8]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP8]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP8]], [[N_MOD_VF]]
; CHECK-NEXT: [[IND_END:%.*]] = add i32 [[X]], [[N_VEC]]
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[Y:%.*]], i32 0
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -195,7 +195,7 @@ define i32 @PR35734(i32 %x, i32 %y) {
; CHECK: middle.block:
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP10:%.*]] = sext i1 [[TMP9]] to i32
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP8]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[X]], [[ENTRY:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/replace-first-order-recurrence-by-versioned-iv.ll b/llvm/test/Transforms/LoopVectorize/replace-first-order-recurrence-by-versioned-iv.ll
index f8e5025c28333..52c2c313db130 100644
--- a/llvm/test/Transforms/LoopVectorize/replace-first-order-recurrence-by-versioned-iv.ll
+++ b/llvm/test/Transforms/LoopVectorize/replace-first-order-recurrence-by-versioned-iv.ll
@@ -8,7 +8,7 @@ define void @replace_first_order_recurrence_with_versioned_iv_for_pointer_use(pt
; CHECK-NEXT: [[X2:%.*]] = ptrtoaddr ptr [[X]] to i64
; CHECK-NEXT: [[Y1:%.*]] = ptrtoaddr ptr [[Y]] to i64
; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N]], i32 1)
-; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[SMAX]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[SMAX]] to i64
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
@@ -168,7 +168,7 @@ define i64 @do_not_replace_first_order_recurrence_with_versioned_iv_for_wide_poi
; CHECK-NEXT: [[X2:%.*]] = ptrtoaddr ptr [[X]] to i64
; CHECK-NEXT: [[Y1:%.*]] = ptrtoaddr ptr [[Y]] to i64
; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N]], i32 1)
-; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[SMAX]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[SMAX]] to i64
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
index 86167582bd12e..26f7fc23d7ef7 100644
--- a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
+++ b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
@@ -21,8 +21,8 @@ define void @reuse_lcssa_phi_for_add_rec1(ptr %head) {
; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_1]] ]
; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ [[IV_2_NEXT]], %[[LOOP_1]] ]
; CHECK-NEXT: [[SRC_2:%.*]] = tail call noalias noundef dereferenceable_or_null(8) ptr @calloc(i64 1, i64 8)
-; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP0]], i32 1)
-; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[IV_2_NEXT]], i32 1)
+; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[IV_2_NEXT]], [[SMIN]]
; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll b/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
index 861d7095d9263..a5ed674d7ddf6 100644
--- a/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
+++ b/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
@@ -34,12 +34,12 @@ define void @test_pr63368(i1 %c, ptr %A) {
; CHECK-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[TMP3]], 255
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label [[SCALAR_PH]], label [[VECTOR_PH2:%.*]]
-; CHECK: vector.ph2:
+; CHECK: vector.ph1:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP2]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[N_VEC]] to i8
; CHECK-NEXT: br label [[VECTOR_BODY3:%.*]]
-; CHECK: vector.body3:
+; CHECK: vector.body2:
; CHECK-NEXT: [[INDEX4:%.*]] = phi i32 [ 0, [[VECTOR_PH2]] ], [ [[INDEX_NEXT5:%.*]], [[VECTOR_BODY3]] ]
; CHECK-NEXT: [[TMP10:%.*]] = trunc i32 [[INDEX4]] to i8
; CHECK-NEXT: [[TMP11:%.*]] = add i8 [[TMP10]], 1
@@ -48,7 +48,7 @@ define void @test_pr63368(i1 %c, ptr %A) {
; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i32 [[INDEX4]], 4
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT5]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK6:%.*]], label [[VECTOR_BODY3]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: middle.block6:
+; CHECK: middle.block5:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_2:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll b/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll
index 71080b9296721..5cf72a642147c 100644
--- a/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll
+++ b/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll
@@ -202,10 +202,6 @@ define void @no_signed_wrap_iv_via_btc(ptr %dst, i32 %N) mustprogress {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[SUB:%.*]] = add i32 [[N]], -100
; CHECK-NEXT: [[SUB4:%.*]] = add i32 [[N]], -99
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], 1
-; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[SUB4]], i32 [[TMP0]])
-; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[SMAX]], 100
-; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], [[N]]
; CHECK-NEXT: br label %[[OUTER:.*]]
; CHECK: [[OUTER_LOOPEXIT:.*]]:
; CHECK-NEXT: br label %[[OUTER]]
@@ -213,6 +209,10 @@ define void @no_signed_wrap_iv_via_btc(ptr %dst, i32 %N) mustprogress {
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], [[EXIT:label %.*]]
; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], 1
+; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[SUB4]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP7:%.*]] = sub i32 [[SMAX]], [[N]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP7]], 100
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP2]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
index c54fe01a844d3..c02847c55f23b 100644
--- a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+++ b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
@@ -984,7 +984,10 @@ define i64 @multi_exit_4_exit_count_with_urem_by_constant_in_latch(ptr %dst, i64
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
-; CHECK-NEXT: [[TMP0:%.*]] = urem i64 [[N]], 42
+; CHECK-NEXT: [[TMP4:%.*]] = udiv i64 [[N]], 42
+; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], -42
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[N]], [[TMP5]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 0)
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[TMP0]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw i64 [[UMIN]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
index d88cb7e79f143..8272e2d9a1542 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
@@ -20,11 +20,12 @@ define void @expand(ptr %src, ptr %dst, i64 %0) {
; CHECK-NEXT: [[SMAX6:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 1000)
; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[SMAX6]], 4
; CHECK-NEXT: [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT: [[SMAX8:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 1000)
-; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[SMAX8]], [[TMP0]]
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP0]], 1
+; CHECK-NEXT: [[SMAX8:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP13]], i64 1000)
+; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[SMAX8]], [[TMP0]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP8]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
index 08a4d3ffefb76..fac74551a8add 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
@@ -66,7 +66,7 @@ define void @s172(i32 noundef %xa, i32 noundef %xb, ptr noundef %a, ptr noundef
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP8]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER13]]
-; CHECK: for.body.preheader14:
+; CHECK: for.body.preheader13:
; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ [[TMP0]], [[VECTOR_MEMCHECK]] ], [ [[TMP0]], [[FOR_BODY_PREHEADER]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
More information about the llvm-commits
mailing list