[llvm] d31b11c - [LV] Make greater use of ScalarEvolution in addDiffRuntimeChecks (#213668)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 01:14:57 PDT 2026
Author: David Sherwood
Date: 2026-08-05T09:14:51+01:00
New Revision: d31b11c260ae09545222743ab5c61841dfe248da
URL: https://github.com/llvm/llvm-project/commit/d31b11c260ae09545222743ab5c61841dfe248da
DIFF: https://github.com/llvm/llvm-project/commit/d31b11c260ae09545222743ab5c61841dfe248da.diff
LOG: [LV] Make greater use of ScalarEvolution in addDiffRuntimeChecks (#213668)
When the loop vectoriser calls addDiffRuntimeChecks it goes to a lot of
effort to avoid generating multiple copies of the VF calculation by
caching the first instance. However, now that ScalarEvolution has a
getElementCount function we can simplify this code significantly,
especially since SCEVs are also implicitly cached. Make greater use of
SCEVs for computation also has the side-effect of improving code quality
in the memory check blocks, which is important when estimating costs of
these checks. You can see this in some tests like
LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
where the threshold for entering the loop has been relaxed.
Ideally, all computation in addDiffRuntimeChecks should be done using
SCEV because there are presumably other folds that can be applied to the
comparisons. However, I'd keep things simple in this PR and deal with
that in a follow-on PR.
Added:
llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop-diff-checks.ll
Modified:
llvm/include/llvm/Transforms/Utils/LoopUtils.h
llvm/lib/Transforms/Utils/LoopUtils.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll
llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 2cca26af577e1..0bf2d866b72bf 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -665,9 +665,10 @@ addRuntimeChecks(Instruction *Loc, Loop *TheLoop,
const SmallVectorImpl<RuntimePointerCheck> &PointerChecks,
SCEVExpander &Expander, bool HoistRuntimeChecks = false);
-LLVM_ABI Value *addDiffRuntimeChecks(
- Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander,
- function_ref<Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC);
+LLVM_ABI Value *addDiffRuntimeChecks(Instruction *Loc,
+ ArrayRef<PointerDiffInfo> Checks,
+ SCEVExpander &Expander, ElementCount VF,
+ unsigned IC);
/// Struct to hold information about a partially invariant condition.
struct IVConditionInfo {
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 1327fb0095647..f5dcdb4ace162 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2278,9 +2278,10 @@ Value *llvm::addRuntimeChecks(
return MemoryRuntimeCheck;
}
-Value *llvm::addDiffRuntimeChecks(
- Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander,
- function_ref<Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC) {
+Value *llvm::addDiffRuntimeChecks(Instruction *Loc,
+ ArrayRef<PointerDiffInfo> Checks,
+ SCEVExpander &Expander, ElementCount VF,
+ unsigned IC) {
LLVMContext &Ctx = Loc->getContext();
IRBuilder ChkBuilder(Ctx, InstSimplifyFolder(Loc->getDataLayout()));
@@ -2292,22 +2293,13 @@ Value *llvm::addDiffRuntimeChecks(
// Map to keep track of created compares, The key is the pair of operands for
// the compare, to allow detecting and re-using redundant compares.
DenseMap<std::pair<Value *, Value *>, Value *> SeenCompares;
- // Cache of (VF * IC * AccessSize) - 1, shared across checks with matching
- // type and IC*AccessSize to avoid emitting duplicate runtime computations.
- DenseMap<std::pair<Type *, unsigned>, Value *> ThresholdCache;
for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze] : Checks) {
assert(IC * AccessSize > 0 &&
"Threshold must be non-zero to use
diff -check");
Type *Ty = SinkStart->getType();
- unsigned ICTimesAccessSize = IC * AccessSize;
- Value *One = ConstantInt::get(Ty, 1);
- Value *&ThresholdMinusOne = ThresholdCache[{Ty, ICTimesAccessSize}];
- if (!ThresholdMinusOne) {
- Value *VFTimesICTimesSize =
- ChkBuilder.CreateMul(GetVF(ChkBuilder, Ty->getScalarSizeInBits()),
- ConstantInt::get(Ty, ICTimesAccessSize));
- ThresholdMinusOne = ChkBuilder.CreateSub(VFTimesICTimesSize, One);
- }
+ const SCEV *TotalAccessSize = SE.getElementCount(Ty, VF * IC * AccessSize);
+ Value *ThresholdMinusOne = Expander.expandCodeFor(
+ SE.getMinusSCEV(TotalAccessSize, SE.getConstant(Ty, 1)), Ty, Loc);
Value *Diff =
Expander.expandCodeFor(SE.getMinusSCEV(SinkStart, SrcStart), Ty, Loc);
@@ -2319,8 +2311,9 @@ Value *llvm::addDiffRuntimeChecks(
// Use (Diff - 1) <u (Threshold - 1), equivalent to 0 < Diff <u Threshold,
// to exclude Diff == 0 (equal pointers are safe).
- IsConflict = ChkBuilder.CreateICmpULT(ChkBuilder.CreateSub(Diff, One),
- ThresholdMinusOne, "
diff .check");
+ IsConflict = ChkBuilder.CreateICmpULT(
+ ChkBuilder.CreateSub(Diff, ConstantInt::get(Ty, 1)), ThresholdMinusOne,
+ "
diff .check");
SeenCompares.insert({{Diff, ThresholdMinusOne}, IsConflict});
if (NeedsFreeze)
IsConflict =
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index de385ae0a4511..3eb6d0530da6d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1657,15 +1657,8 @@ class GeneratedRTChecks {
auto DiffChecks = RtPtrChecking.getDiffChecks();
if (DiffChecks) {
- Value *RuntimeVF = nullptr;
MemRuntimeCheckCond = addDiffRuntimeChecks(
- MemCheckBlock->getTerminator(), *DiffChecks, MemCheckExp,
- [VF, &RuntimeVF](IRBuilderBase &B, unsigned Bits) {
- if (!RuntimeVF)
- RuntimeVF = getRuntimeVF(B, B.getIntNTy(Bits), VF);
- return RuntimeVF;
- },
- IC);
+ MemCheckBlock->getTerminator(), *DiffChecks, MemCheckExp, VF, IC);
} else {
MemRuntimeCheckCond = addRuntimeChecks(
MemCheckBlock->getTerminator(), L, RtPtrChecking.getChecks(),
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
index 87bdd1f975423..6abc80385e9dd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
@@ -265,8 +265,8 @@ define void @alias_mask_reverse_iterate(ptr noalias %ptrA, ptr %ptrB, ptr %ptrC,
; CHECK-TF-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK-TF: [[VECTOR_MEMCHECK]]:
; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; CHECK-TF-NEXT: [[TMP5:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-TF-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-TF-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], -1
; CHECK-TF-NEXT: [[TMP2:%.*]] = sub i64 [[PTRB1]], [[PTRC2]]
; CHECK-TF-NEXT: [[TMP6:%.*]] = sub i64 [[TMP2]], 1
; CHECK-TF-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 09852d7951c09..48c6b259c8b78 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -1137,8 +1137,8 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; DEFAULT-NEXT: [[TMP8:%.*]] = sub i64 [[TMP5]], 1
+; DEFAULT-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; DEFAULT-NEXT: [[TMP8:%.*]] = add i64 [[TMP5]], -1
; DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 [[C1]], [[A2]]
; DEFAULT-NEXT: [[TMP11:%.*]] = sub i64 [[TMP6]], 1
; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP8]]
@@ -1192,8 +1192,8 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
; PRED-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; PRED: [[VECTOR_MEMCHECK]]:
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; PRED-NEXT: [[TMP5:%.*]] = sub i64 [[TMP2]], 1
+; PRED-NEXT: [[TMP2:%.*]] = shl i64 [[TMP1]], 2
+; PRED-NEXT: [[TMP5:%.*]] = add i64 [[TMP2]], -1
; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[C1]], [[A2]]
; PRED-NEXT: [[TMP6:%.*]] = sub i64 [[TMP3]], 1
; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 33c781b99a9bc..f4a94f552bd57 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -20,9 +20,8 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
-; DEFAULT-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; DEFAULT-NEXT: [[TMP27:%.*]] = sub i64 [[TMP5]], 1
+; DEFAULT-NEXT: [[TMP9:%.*]] = shl i64 [[TMP3]], 4
+; DEFAULT-NEXT: [[TMP27:%.*]] = add i64 [[TMP9]], -1
; DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 [[DST1]], [[SRC2]]
; DEFAULT-NEXT: [[TMP14:%.*]] = sub i64 [[TMP6]], 1
; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP14]], [[TMP27]]
@@ -126,8 +125,8 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; PRED-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; PRED: [[VECTOR_MEMCHECK]]:
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
-; PRED-NEXT: [[TMP7:%.*]] = sub i64 [[TMP2]], 1
+; PRED-NEXT: [[TMP2:%.*]] = shl i64 [[TMP1]], 3
+; PRED-NEXT: [[TMP7:%.*]] = add i64 [[TMP2]], -1
; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[DST1]], [[SRC2]]
; PRED-NEXT: [[TMP6:%.*]] = sub i64 [[TMP3]], 1
; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP7]]
@@ -599,11 +598,11 @@ define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; DEFAULT-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
-; DEFAULT-NEXT: [[IV_CONV:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[IV_EXT:%.*]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[IV_CONV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_EXT:%.*]], %[[LOOP]] ]
; DEFAULT-NEXT: [[GEP:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[IV_CONV]], i32 2
; DEFAULT-NEXT: store i32 0, ptr [[GEP]], align 8
; DEFAULT-NEXT: [[IV_1_NEXT]] = add i32 [[IV_1]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll
index 256708bbfd0f6..7a0893e23ab38 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll
@@ -123,9 +123,8 @@ define void @struct_return_f32_widen_rt_checks(ptr %in, ptr writeonly %out_a, pt
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4
-; CHECK-NEXT: [[TMP23:%.*]] = sub i64 [[TMP2]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP1]], -1
; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[OUT_B1]], [[OUT_A2]]
; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[TMP3]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP14]], [[TMP23]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
index 4245b1d6ffef7..ae3bd1d54a9f8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
@@ -21,9 +21,8 @@ define void @fneg(ptr nocapture noundef writeonly %d, ptr nocapture noundef read
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4
-; CHECK-NEXT: [[TMP10:%.*]] = sub i64 [[TMP4]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 5
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw i64 [[TMP3]], -1
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[D1]], [[S2]]
; CHECK-NEXT: [[TMP9:%.*]] = sub i64 [[TMP5]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP9]], [[TMP10]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
index 44306ec3c38d9..c8961090bfea5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
@@ -22,9 +22,8 @@ define void @multiple_exits_unique_exit_block(ptr %A, ptr %B, i32 %N) #0 {
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[TMP5]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP3]], 5
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP7]], -1
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[TMP6]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
@@ -92,9 +91,8 @@ define i32 @multiple_exits_multiple_exit_blocks(ptr %A, ptr %B, i32 %N) #0 {
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[TMP5]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP3]], 5
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP7]], -1
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[TMP6]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop-
diff -checks.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop-
diff -checks.ll
new file mode 100644
index 0000000000000..5f3f9d6fc958d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-nested-loop-
diff -checks.ll
@@ -0,0 +1,84 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 6
+; RUN: opt -p loop-vectorize -mattr=+sve -S < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @foo(ptr %dst, ptr %src, i32 %m, i32 %n) vscale_range(1,16) {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i32 [[M:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[TMP11]], -1
+; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-NEXT: br label %[[OUTER_LOOP:.*]]
+; CHECK: [[OUTER_LOOP]]:
+; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i32 [ [[OUTER_IV_NEXT:%.*]], %[[INNER_EXIT:.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[OUTER_IV]], 3
+; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP13]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP2]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP15:%.*]] = sub i64 [[TMP14]], 1
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = icmp ult i64 [[TMP15]], [[TMP12]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP13]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP4]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[ADD]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP1]], i64 [[TMP3]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = mul nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = mul nsw <vscale x 4 x i32> [[WIDE_LOAD2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP7]], ptr [[TMP5]], align 4
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP8]], ptr [[TMP10]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[INNER_EXIT]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ %wide.trip.count = zext nneg i32 %n to i64
+ br label %outer.loop
+
+outer.loop:
+ %outer.iv = phi i32 [ %outer.iv.next, %inner.exit ], [ 0, %entry ]
+ %add = add nuw nsw i32 %outer.iv, 3
+ br label %inner.loop
+
+inner.loop:
+ %iv = phi i64 [ 0, %outer.loop ], [ %iv.next, %inner.loop ]
+ %arrayidx = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
+ %ld1 = load i32, ptr %arrayidx, align 4
+ %mul = mul nsw i32 %ld1, %add
+ %arrayidx6 = getelementptr inbounds nuw [4 x i8], ptr %dst, i64 %iv
+ store i32 %mul, ptr %arrayidx6, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %wide.trip.count
+ br i1 %exitcond.not, label %inner.exit, label %inner.loop
+
+inner.exit:
+ %outer.iv.next = add nuw nsw i32 %outer.iv, 1
+ %exitcond2.not = icmp eq i32 %outer.iv.next, %m
+ br i1 %exitcond2.not, label %exit, label %outer.loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
index 0f4340731ee2f..30f7b057e6708 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
@@ -15,14 +15,13 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr
; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 22)
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 20)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
-; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 16
-; CHECK-NEXT: [[TMP21:%.*]] = sub i64 [[TMP5]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP3]], 5
+; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[TMP5]], -1
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[DST_21]], [[DST_12]]
; CHECK-NEXT: [[TMP20:%.*]] = sub i64 [[TMP6]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP20]], [[TMP21]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
index 82fe12a43f842..d1d1aee639e0c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
@@ -86,9 +86,8 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 {
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP5]], 16
-; CHECK-NEXT: [[TMP16:%.*]] = sub i64 [[TMP3]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 7
+; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[TMP3]], -1
; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP4]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP16]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 2ec48b25af5b5..e0a1b2bb6a03e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -636,9 +636,8 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP3]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP6]], 4
-; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[TMP4]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; CHECK-NEXT: [[TMP17:%.*]] = add nsw i64 [[TMP4]], -1
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[ARRAY1]], [[INDICES2]]
; CHECK-NEXT: [[TMP10:%.*]] = sub i64 [[TMP5]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP17]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll b/llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll
index 45807888dafee..91276d0c877c7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll
@@ -15,8 +15,8 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP1]], -1
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[C1]], [[B2]]
; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP2]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP6]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll b/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
index 67df7c2aba896..584051589aa27 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
@@ -12,14 +12,13 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 2
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 19)
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 17)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP15]], 4
-; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[TMP16]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP4]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP12]]
@@ -71,14 +70,13 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
; ZVFHMIN-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 19)
+; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 17)
; ZVFHMIN-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
; ZVFHMIN-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; ZVFHMIN-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; ZVFHMIN-NEXT: [[TMP14:%.*]] = sub i64 [[TMP5]], 1
+; ZVFHMIN-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP14:%.*]] = add i64 [[TMP4]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP12:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
@@ -152,14 +150,13 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 2
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 19)
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 17)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP15]], 4
-; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[TMP16]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP4]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP12]]
@@ -211,14 +208,13 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
; ZVFHMIN-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 19)
+; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 17)
; ZVFHMIN-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
; ZVFHMIN-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; ZVFHMIN-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; ZVFHMIN-NEXT: [[TMP14:%.*]] = sub i64 [[TMP5]], 1
+; ZVFHMIN-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP14:%.*]] = add i64 [[TMP4]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP12:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
@@ -292,14 +288,13 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 19)
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 17)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP15]], 2
-; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[TMP16]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP4]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP12]]
@@ -351,14 +346,13 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
; ZVFHMIN-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 19)
+; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 17)
; ZVFHMIN-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
; ZVFHMIN-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
-; ZVFHMIN-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; ZVFHMIN-NEXT: [[TMP14:%.*]] = sub i64 [[TMP5]], 1
+; ZVFHMIN-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP14:%.*]] = add i64 [[TMP4]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP12:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
@@ -432,14 +426,13 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 19)
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 17)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP15]], 2
-; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[TMP16]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP4]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP12]]
@@ -491,14 +484,13 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
; ZVFHMIN-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 19)
+; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 17)
; ZVFHMIN-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
; ZVFHMIN-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
-; ZVFHMIN-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; ZVFHMIN-NEXT: [[TMP14:%.*]] = sub i64 [[TMP5]], 1
+; ZVFHMIN-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP14:%.*]] = add i64 [[TMP4]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP12:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
@@ -573,9 +565,8 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
-; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[TMP16]], 2
-; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP18]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP3]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
@@ -630,9 +621,8 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
-; ZVFHMIN-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; ZVFHMIN-NEXT: [[TMP11:%.*]] = sub i64 [[TMP5]], 1
+; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP11:%.*]] = add i64 [[TMP1]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
@@ -709,9 +699,8 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
-; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[TMP16]], 2
-; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP18]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP3]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
@@ -766,9 +755,8 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
-; ZVFHMIN-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; ZVFHMIN-NEXT: [[TMP11:%.*]] = sub i64 [[TMP5]], 1
+; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP11:%.*]] = add i64 [[TMP1]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 100ecc172fab0..b395e69868507 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -162,9 +162,8 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV64-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; RV64: [[VECTOR_MEMCHECK]]:
; RV64-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT: [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
-; RV64-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4
-; RV64-NEXT: [[TMP16:%.*]] = sub i64 [[TMP13]], 1
+; RV64-NEXT: [[TMP10:%.*]] = shl i64 [[TMP11]], 4
+; RV64-NEXT: [[TMP16:%.*]] = add i64 [[TMP10]], -1
; RV64-NEXT: [[TMP14:%.*]] = sub i64 [[B1]], [[A2]]
; RV64-NEXT: [[TMP15:%.*]] = sub i64 [[TMP14]], 1
; RV64-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP15]], [[TMP16]]
@@ -215,9 +214,8 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV32-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; RV32: [[VECTOR_MEMCHECK]]:
; RV32-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; RV32-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
-; RV32-NEXT: [[TMP5:%.*]] = mul i32 [[TMP4]], 4
-; RV32-NEXT: [[TMP8:%.*]] = sub i32 [[TMP5]], 1
+; RV32-NEXT: [[TMP2:%.*]] = shl i32 [[TMP3]], 4
+; RV32-NEXT: [[TMP8:%.*]] = add i32 [[TMP2]], -1
; RV32-NEXT: [[TMP6:%.*]] = sub i32 [[B1]], [[A2]]
; RV32-NEXT: [[TMP7:%.*]] = sub i32 [[TMP6]], 1
; RV32-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i32 [[TMP7]], [[TMP8]]
@@ -280,9 +278,8 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV64-UF2-NEXT: br i1 [[TMP10]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; RV64-UF2: [[VECTOR_MEMCHECK]]:
; RV64-UF2-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT: [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
-; RV64-UF2-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 8
-; RV64-UF2-NEXT: [[TMP15:%.*]] = sub i64 [[TMP13]], 1
+; RV64-UF2-NEXT: [[TMP12:%.*]] = shl i64 [[TMP11]], 5
+; RV64-UF2-NEXT: [[TMP15:%.*]] = add i64 [[TMP12]], -1
; RV64-UF2-NEXT: [[TMP14:%.*]] = sub i64 [[B1]], [[A2]]
; RV64-UF2-NEXT: [[TMP16:%.*]] = sub i64 [[TMP14]], 1
; RV64-UF2-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP16]], [[TMP15]]
@@ -381,9 +378,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV64-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; RV64: [[VECTOR_MEMCHECK]]:
; RV64-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT: [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
-; RV64-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4
-; RV64-NEXT: [[TMP16:%.*]] = sub i64 [[TMP13]], 1
+; RV64-NEXT: [[TMP10:%.*]] = shl i64 [[TMP11]], 4
+; RV64-NEXT: [[TMP16:%.*]] = add i64 [[TMP10]], -1
; RV64-NEXT: [[TMP14:%.*]] = sub i64 [[B1]], [[A2]]
; RV64-NEXT: [[TMP15:%.*]] = sub i64 [[TMP14]], 1
; RV64-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP15]], [[TMP16]]
@@ -434,9 +430,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV32-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; RV32: [[VECTOR_MEMCHECK]]:
; RV32-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; RV32-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
-; RV32-NEXT: [[TMP5:%.*]] = mul i32 [[TMP4]], 4
-; RV32-NEXT: [[TMP8:%.*]] = sub i32 [[TMP5]], 1
+; RV32-NEXT: [[TMP2:%.*]] = shl i32 [[TMP3]], 4
+; RV32-NEXT: [[TMP8:%.*]] = add i32 [[TMP2]], -1
; RV32-NEXT: [[TMP6:%.*]] = sub i32 [[B1]], [[A2]]
; RV32-NEXT: [[TMP7:%.*]] = sub i32 [[TMP6]], 1
; RV32-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i32 [[TMP7]], [[TMP8]]
@@ -499,9 +494,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV64-UF2-NEXT: br i1 [[TMP10]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; RV64-UF2: [[VECTOR_MEMCHECK]]:
; RV64-UF2-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT: [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
-; RV64-UF2-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 8
-; RV64-UF2-NEXT: [[TMP15:%.*]] = sub i64 [[TMP13]], 1
+; RV64-UF2-NEXT: [[TMP12:%.*]] = shl i64 [[TMP11]], 5
+; RV64-UF2-NEXT: [[TMP15:%.*]] = add i64 [[TMP12]], -1
; RV64-UF2-NEXT: [[TMP14:%.*]] = sub i64 [[B1]], [[A2]]
; RV64-UF2-NEXT: [[TMP16:%.*]] = sub i64 [[TMP14]], 1
; RV64-UF2-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP16]], [[TMP15]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 4f9a0697608b6..d2972748af795 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -643,9 +643,8 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; NOSTRIDED-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NOSTRIDED: [[VECTOR_MEMCHECK]]:
; NOSTRIDED-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NOSTRIDED-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NOSTRIDED-NEXT: [[TMP8:%.*]] = sub i64 [[TMP5]], 1
+; NOSTRIDED-NEXT: [[TMP1:%.*]] = shl i64 [[TMP3]], 4
+; NOSTRIDED-NEXT: [[TMP8:%.*]] = add i64 [[TMP1]], -1
; NOSTRIDED-NEXT: [[TMP6:%.*]] = sub i64 [[P21]], [[P3]]
; NOSTRIDED-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], 1
; NOSTRIDED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP8]]
@@ -691,7 +690,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; NOSTRIDED-UF2-NEXT: [[P21:%.*]] = ptrtoaddr ptr [[P2]] to i64
; NOSTRIDED-UF2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NOSTRIDED-UF2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; NOSTRIDED-UF2-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 16)
+; NOSTRIDED-UF2-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 14)
; NOSTRIDED-UF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
; NOSTRIDED-UF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; NOSTRIDED-UF2: [[VECTOR_SCEVCHECK]]:
@@ -699,9 +698,8 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; NOSTRIDED-UF2-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; NOSTRIDED-UF2: [[VECTOR_MEMCHECK]]:
; NOSTRIDED-UF2-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-UF2-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; NOSTRIDED-UF2-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 8
-; NOSTRIDED-UF2-NEXT: [[TMP15:%.*]] = sub i64 [[TMP4]], 1
+; NOSTRIDED-UF2-NEXT: [[TMP4:%.*]] = shl i64 [[TMP2]], 5
+; NOSTRIDED-UF2-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
; NOSTRIDED-UF2-NEXT: [[TMP5:%.*]] = sub i64 [[P21]], [[P3]]
; NOSTRIDED-UF2-NEXT: [[TMP10:%.*]] = sub i64 [[TMP5]], 1
; NOSTRIDED-UF2-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
index a95b34560038f..4a5c127b31c90 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
@@ -17,8 +17,8 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -68,8 +68,8 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -135,8 +135,8 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -186,8 +186,8 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -253,8 +253,8 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -304,8 +304,8 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -371,8 +371,8 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -422,8 +422,8 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -489,8 +489,8 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -540,8 +540,8 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -607,8 +607,8 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -658,8 +658,8 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -725,8 +725,8 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -776,8 +776,8 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -843,8 +843,8 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -894,8 +894,8 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -961,8 +961,8 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -1012,8 +1012,8 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -1079,8 +1079,8 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -1130,8 +1130,8 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -1197,8 +1197,8 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -1248,8 +1248,8 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -1315,8 +1315,8 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -1366,8 +1366,8 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -1433,8 +1433,8 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -1484,8 +1484,8 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; NO-VP-NEXT: [[TMP13:%.*]] = sub i64 [[TMP3]], 1
+; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -1553,9 +1553,8 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4
-; IF-EVL-NEXT: [[TMP5:%.*]] = sub i64 [[TMP2]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP5]]
@@ -1605,9 +1604,8 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
@@ -1673,9 +1671,8 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4
-; IF-EVL-NEXT: [[TMP5:%.*]] = sub i64 [[TMP2]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP5]]
@@ -1725,9 +1722,8 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
@@ -1793,9 +1789,8 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4
-; IF-EVL-NEXT: [[TMP5:%.*]] = sub i64 [[TMP2]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP5]]
@@ -1845,9 +1840,8 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
@@ -1913,9 +1907,8 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4
-; IF-EVL-NEXT: [[TMP5:%.*]] = sub i64 [[TMP2]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP5]]
@@ -1965,9 +1958,8 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
@@ -2086,9 +2078,8 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4
-; IF-EVL-NEXT: [[TMP5:%.*]] = sub i64 [[TMP2]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP5]]
@@ -2138,9 +2129,8 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll
index bfad85a7f9e67..ba1edbdd35fb3 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll
@@ -17,9 +17,8 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT: [[TMP23:%.*]] = mul i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = sub i64 [[TMP23]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP4]], 4
+; IF-EVL-NEXT: [[TMP7:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP6:%.*]] = sub i64 [[TMP24]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP7]]
@@ -78,9 +77,8 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP19:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP19:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP18:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP18]], [[TMP19]]
@@ -158,9 +156,8 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT: [[TMP23:%.*]] = mul i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = sub i64 [[TMP23]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP4]], 4
+; IF-EVL-NEXT: [[TMP7:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP6:%.*]] = sub i64 [[TMP24]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP7]]
@@ -219,9 +216,8 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP19:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP19:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP18:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP18]], [[TMP19]]
@@ -299,9 +295,8 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT: [[TMP23:%.*]] = mul i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = sub i64 [[TMP23]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP4]], 4
+; IF-EVL-NEXT: [[TMP7:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP6:%.*]] = sub i64 [[TMP24]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP7]]
@@ -360,9 +355,8 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP19:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP19:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP18:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP18]], [[TMP19]]
@@ -440,9 +434,8 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT: [[TMP23:%.*]] = mul i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = sub i64 [[TMP23]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP4]], 4
+; IF-EVL-NEXT: [[TMP7:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP6:%.*]] = sub i64 [[TMP24]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP7]]
@@ -501,9 +494,8 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP19:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP19:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP18:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP18]], [[TMP19]]
@@ -581,9 +573,8 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT: [[TMP20:%.*]] = mul i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = sub i64 [[TMP20]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP4]], 4
+; IF-EVL-NEXT: [[TMP7:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP21:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP6:%.*]] = sub i64 [[TMP21]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP7]]
@@ -633,9 +624,8 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP11:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP15]]
@@ -702,9 +692,8 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP9:%.*]] = sub i64 [[TMP6]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP4]], 4
+; IF-EVL-NEXT: [[TMP9:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP7:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP9]]
@@ -754,9 +743,8 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP11:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP15]]
@@ -823,9 +811,8 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT: [[TMP23:%.*]] = mul i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = sub i64 [[TMP23]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP4]], 4
+; IF-EVL-NEXT: [[TMP7:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP6:%.*]] = sub i64 [[TMP24]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP7]]
@@ -879,9 +866,8 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP17:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP17:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP11:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP17]]
@@ -954,9 +940,8 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT: [[TMP23:%.*]] = mul i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = sub i64 [[TMP23]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP4]], 4
+; IF-EVL-NEXT: [[TMP7:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP6:%.*]] = sub i64 [[TMP24]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP7]]
@@ -1010,9 +995,8 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP17:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP17:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP11:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP17]]
@@ -1085,9 +1069,8 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT: [[TMP20:%.*]] = mul i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = sub i64 [[TMP20]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP4]], 4
+; IF-EVL-NEXT: [[TMP7:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP21:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP6:%.*]] = sub i64 [[TMP21]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP7]]
@@ -1137,9 +1120,8 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP11:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP11:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
index 40f7abcfcd9f9..08b3acd4c1a55 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
@@ -596,9 +596,8 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4
-; IF-EVL-NEXT: [[TMP9:%.*]] = sub i64 [[TMP7]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP5]], 4
+; IF-EVL-NEXT: [[TMP9:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP8]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP9]]
@@ -648,9 +647,8 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP11:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP11:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
@@ -716,9 +714,8 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4
-; IF-EVL-NEXT: [[TMP9:%.*]] = sub i64 [[TMP7]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP5]], 4
+; IF-EVL-NEXT: [[TMP9:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP8]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP9]]
@@ -768,9 +765,8 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP11:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP11:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
@@ -836,9 +832,8 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4
-; IF-EVL-NEXT: [[TMP9:%.*]] = sub i64 [[TMP7]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP5]], 4
+; IF-EVL-NEXT: [[TMP9:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP8]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP9]]
@@ -888,9 +883,8 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP11:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP11:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
@@ -956,9 +950,8 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; IF-EVL-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4
-; IF-EVL-NEXT: [[TMP9:%.*]] = sub i64 [[TMP7]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP5]], 4
+; IF-EVL-NEXT: [[TMP9:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP8]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP9]]
@@ -1008,9 +1001,8 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; NO-VP-NEXT: [[TMP11:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP11:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
@@ -1076,9 +1068,8 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
-; IF-EVL-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 8
-; IF-EVL-NEXT: [[TMP9:%.*]] = sub i64 [[TMP7]], 1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP5]], 4
+; IF-EVL-NEXT: [[TMP9:%.*]] = add i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP8]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP9]]
@@ -1123,14 +1114,13 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
; NO-VP-NEXT: [[A1:%.*]] = ptrtoaddr ptr [[A]] to i64
; NO-VP-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP10]], 1
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP9]], i64 20)
+; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP9]], i64 16)
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
-; NO-VP-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; NO-VP-NEXT: [[TMP15:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[A1]], [[B2]]
; NO-VP-NEXT: [[TMP11:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP11]], [[TMP15]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
index 0cd887ad8fe95..87aa3f9a91463 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
@@ -25,9 +25,8 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
; CHECK-VF4UF1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK-VF4UF1: [[VECTOR_MEMCHECK]]:
; CHECK-VF4UF1-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; CHECK-VF4UF1-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4
-; CHECK-VF4UF1-NEXT: [[TMP13:%.*]] = sub i64 [[TMP7]], 1
+; CHECK-VF4UF1-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 4
+; CHECK-VF4UF1-NEXT: [[TMP13:%.*]] = add i64 [[TMP6]], -1
; CHECK-VF4UF1-NEXT: [[TMP8:%.*]] = add i64 [[B1]], -4
; CHECK-VF4UF1-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], [[A2]]
; CHECK-VF4UF1-NEXT: [[TMP12:%.*]] = sub i64 [[TMP9]], 1
@@ -89,9 +88,8 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
; CHECK-VF4UF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK-VF4UF2: [[VECTOR_MEMCHECK]]:
; CHECK-VF4UF2-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; CHECK-VF4UF2-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 8
-; CHECK-VF4UF2-NEXT: [[TMP19:%.*]] = sub i64 [[TMP7]], 1
+; CHECK-VF4UF2-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 5
+; CHECK-VF4UF2-NEXT: [[TMP19:%.*]] = add i64 [[TMP6]], -1
; CHECK-VF4UF2-NEXT: [[TMP8:%.*]] = add i64 [[B1]], -4
; CHECK-VF4UF2-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], [[A2]]
; CHECK-VF4UF2-NEXT: [[TMP13:%.*]] = sub i64 [[TMP9]], 1
More information about the llvm-commits
mailing list