[flang-commits] [flang] [llvm] [VPlan] Rework nsw-handling in inductions (PR #230060)
Ramkumar Ramachandra via flang-commits
flang-commits at lists.llvm.org
Mon Oct 26 03:46:16 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/230060
>From 26fd8fbf226bcda35785b7a28f44e0c834399605 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 14 Oct 2026 15:49:57 +0100
Subject: [PATCH 1/2] [LV] Add pre-commit induction-wrapflags test
---
.../LoopVectorize/induction-wrapflags.ll | 50 +++++++++++++++++++
1 file changed, 50 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/induction-wrapflags.ll b/llvm/test/Transforms/LoopVectorize/induction-wrapflags.ll
index 8be1725227b2b7..230d3c1968c647 100644
--- a/llvm/test/Transforms/LoopVectorize/induction-wrapflags.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction-wrapflags.ll
@@ -512,5 +512,55 @@ exit:
ret void
}
+define void @add_induction_nsw_constant_step_lane_offset_overflow(ptr noalias %dst, i32 %start, i32 %n) {
+; CHECK-LABEL: define void @add_induction_nsw_constant_step_lane_offset_overflow(
+; CHECK-SAME: ptr noalias [[DST:%.*]], i32 [[START:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[N_VEC]], -1073741830
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[START]], [[TMP3]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[START]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 -1073741830, i32 2147483636, i32 1073741806>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 -24)
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %f = phi i32 [ %start, %entry ], [ %f.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %dst, i32 %iv
+ store i32 %f, ptr %gep
+ %f.next = add nsw i32 %f, -1073741830
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.interleave.count", i32 2}
>From 975932f08ba031f045a1bce6c686c8e83724fcc3 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 8 Oct 2026 09:19:19 +0100
Subject: [PATCH 2/2] [VPlan] Rework nsw-handling in inductions
Fix a miscompile by not constructing add-inductions with nsw on
signed-min step, as verified by llubi.
Ref: dtcxzyw/llvm-autoreduce#512.
Fixes #227866.
---
flang/test/Integration/HLFIR/unroll-loops.fir | 2 +-
flang/test/Integration/unroll-loops.f90 | 2 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 55 ++++++++++--
.../Vectorize/VPlanConstruction.cpp | 5 +-
.../Transforms/Vectorize/VPlanLowering.cpp | 31 +++++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 7 +-
.../Transforms/Vectorize/VPlanTransforms.h | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 21 +++--
.../AArch64/alias-mask-uniforms.ll | 2 +-
.../LoopVectorize/AArch64/cmp_cost.ll | 30 +++----
.../AArch64/conditional-scalar-assignment.ll | 4 +-
.../AArch64/discarded-interleave-group.ll | 2 +-
.../AArch64/epilog-iv-select-cmp.ll | 2 +-
.../epilog-vectorization-widen-inductions.ll | 2 +-
.../AArch64/fully-unrolled-cost.ll | 2 +-
.../AArch64/outer_loop_prefer_scalable.ll | 4 +-
.../AArch64/pr60831-sve-inv-store-crash.ll | 4 +-
.../AArch64/sve-induction-wrapflags.ll | 12 +--
.../LoopVectorize/AArch64/sve-inductions.ll | 2 +-
.../sve-interleaved-masked-accesses.ll | 28 +++---
.../ARM/mve-gather-scatter-tailpred.ll | 8 +-
.../RISCV/dbg-tail-folding-by-evl.ll | 8 +-
.../RISCV/evl-compatible-loops.ll | 4 +-
.../LoopVectorize/RISCV/iv-select-cmp.ll | 8 +-
.../RISCV/tail-folding-cast-intrinsics.ll | 9 +-
.../tail-folding-fixed-order-recurrence.ll | 6 +-
.../RISCV/tail-folding-reduction.ll | 8 +-
.../VPlan/AArch64/call-decisions.ll | 4 +-
.../AArch64/scalarize-after-call-widening.ll | 4 +-
.../AArch64/vplan-memory-op-decisions.ll | 52 +++++------
.../VPlan/RISCV/reductions-evl.ll | 2 +-
.../VPlan/early_exit_with_stores_vplan.ll | 8 +-
.../LoopVectorize/VPlan/find-last.ll | 2 +-
...-order-recurrence-sink-replicate-region.ll | 6 +-
.../LoopVectorize/VPlan/for-phi-ordering.ll | 2 +-
.../LoopVectorize/VPlan/icmp-uniforms.ll | 6 +-
.../LoopVectorize/VPlan/predicator.ll | 66 +++++++-------
.../LoopVectorize/VPlan/print-attributes.ll | 6 +-
.../LoopVectorize/VPlan/print-calls.ll | 8 +-
.../VPlan/simplify-reverse-reverse.ll | 2 +-
.../VPlan/tail-folding-multiuse-reductions.ll | 2 +-
.../VPlan/vplan-based-stride-mv.ll | 88 +++++++++----------
.../VPlan/vplan-iv-transforms.ll | 6 +-
.../VPlan/vplan-narrow-iv-users.ll | 6 +-
.../VPlan/vplan-printing-alias-mask.ll | 44 +++++-----
.../VPlan/vplan-printing-flags.ll | 4 +-
.../VPlan/vplan-printing-intrinsic.ll | 4 +-
.../vplan-printing-reductions-tail-folded.ll | 2 +-
.../VPlan/vplan-printing-reductions.ll | 2 +-
.../LoopVectorize/VPlan/vplan-printing.ll | 20 ++---
.../VPlan/vplan-scev-address-idioms.ll | 40 ++++-----
.../VPlan/vplan-sink-scalars-and-merge.ll | 38 ++++----
.../LoopVectorize/VPlan/widen_mem_idioms.ll | 4 +-
.../X86/CostModel/vpinstruction-cost.ll | 24 ++---
.../X86/fixed-order-recurrence.ll | 2 +-
.../x86-interleaved-accesses-masked-group.ll | 16 ++--
...86-interleaved-store-accesses-with-gaps.ll | 18 ++--
.../Transforms/LoopVectorize/alias-mask.ll | 2 +-
.../LoopVectorize/find-last-iv-interleave.ll | 4 +-
...find-last-iv-sinkable-expr-tail-folding.ll | 12 +--
.../find-last-iv-sinkable-expr.ll | 6 +-
.../LoopVectorize/first-order-recurrence.ll | 2 +-
.../LoopVectorize/force-instruction-cost.ll | 8 +-
.../LoopVectorize/induction-cost.ll | 60 ++++++-------
.../LoopVectorize/induction-step.ll | 16 ++--
.../LoopVectorize/induction-wrapflags.ll | 8 +-
.../Transforms/LoopVectorize/induction.ll | 66 +++++++-------
.../LoopVectorize/iv-select-cmp-decreasing.ll | 18 ++--
.../iv-select-cmp-non-const-iv-start.ll | 6 +-
.../Transforms/LoopVectorize/iv-select-cmp.ll | 24 ++---
.../Transforms/LoopVectorize/loop-form.ll | 2 +-
.../LoopVectorize/memdep-fold-tail.ll | 2 +-
llvm/test/Transforms/LoopVectorize/optsize.ll | 16 ++--
.../remove-redundant-trip-count-scev.ll | 6 +-
.../LoopVectorize/scalable-assume.ll | 2 +-
.../LoopVectorize/scalable-inductions.ll | 10 +--
.../Transforms/LoopVectorize/struct-return.ll | 2 +-
.../tail-folding-argmin-reduction.ll | 24 ++++-
.../tail-folding-masked-mem-opts.ll | 2 +-
.../vplan-based-stride-mv-btc.ll | 12 +--
82 files changed, 577 insertions(+), 496 deletions(-)
diff --git a/flang/test/Integration/HLFIR/unroll-loops.fir b/flang/test/Integration/HLFIR/unroll-loops.fir
index 45b15b7bd5bd78..99b29e41d91c75 100644
--- a/flang/test/Integration/HLFIR/unroll-loops.fir
+++ b/flang/test/Integration/HLFIR/unroll-loops.fir
@@ -37,7 +37,7 @@ func.func @unroll(%arg0: !fir.ref<!fir.array<1000 x index>> {fir.bindc_name = "a
// NO-UNROLL-NEXT: %[[NIV:.*]] = add nuw i64 %{{.*}}, 2
// NO-UNROLL-NEXT: %[[NVIND]] = add nuw nsw <2 x i64> %[[VIND]], splat (i64 2)
- // UNROLL-NEXT: %[[VIND1:.*]] = add nuw nsw <2 x i64> %[[VIND]], splat (i64 2)
+ // UNROLL-NEXT: %[[VIND1:.*]] = add nuw <2 x i64> %[[VIND]], splat (i64 2)
// UNROLL-NEXT: %[[GEP0:.*]] = getelementptr [8 x i8], ptr %[[ARG0]], i64 %[[IND]]
// UNROLL-NEXT: %[[GEP1:.*]] = getelementptr i8, ptr %[[GEP0]], i64 16
// UNROLL-NEXT: store <2 x i64> %[[VIND]], ptr %[[GEP0]]
diff --git a/flang/test/Integration/unroll-loops.f90 b/flang/test/Integration/unroll-loops.f90
index 0ace460dbfe81b..7eccd386da3c83 100644
--- a/flang/test/Integration/unroll-loops.f90
+++ b/flang/test/Integration/unroll-loops.f90
@@ -27,7 +27,7 @@ subroutine unroll(a)
! NO-UNROLL-NEXT: %[[NIV:.*]] = add nuw i64 %{{.*}}, 2
! NO-UNROLL-NEXT: %[[NVIND]] = add nuw nsw <2 x i64> %[[VIND]], splat (i64 2)
!
- ! UNROLL-NEXT: %[[VIND1:.*]] = add nuw nsw <2 x i64> %[[VIND]], splat (i64 2)
+ ! UNROLL-NEXT: %[[VIND1:.*]] = add nuw <2 x i64> %[[VIND]], splat (i64 2)
! UNROLL-NEXT: %[[GEP0:.*]] = getelementptr [8 x i8], ptr %[[ARG0]], i64 %[[IND]]
! UNROLL-NEXT: %[[GEP1:.*]] = getelementptr i8, ptr %[[GEP0]], i64 16
! UNROLL-NEXT: store <2 x i64> %[[VIND]], ptr %[[GEP0]]
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 67dcdf8a035ca1..bdb4d6be17af78 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5539,7 +5539,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, BestVPlan);
- RUN_VPLAN_PASS(VPlanTransforms::convertToConcreteRecipes, BestVPlan);
+ RUN_VPLAN_PASS(VPlanTransforms::convertToConcreteRecipes, BestVPlan,
+ BestVF * BestUF);
// Convert the exit condition to AVLNext == 0 for EVL tail folded loops.
RUN_VPLAN_PASS(VPlanTransforms::convertEVLExitCond, BestVPlan);
// Regions are dissolved after optimizing for VF and UF, which completely
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 46b1c5d08010c3..5f9ddf5c35bb72 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -703,14 +703,51 @@ class VPIRFlags {
};
public:
+ /// Enum to represent presence of no-signed-wrap, as well as speculative
+ /// version that can be turned into the concrete version late, when VF and IC
+ /// are available.
+ enum class VPNoSignedWrap {
+ None = 0,
+ ConcreteNSW = (1 << 0),
+ SpeculativeNSW = (1 << 1)
+ };
+
struct WrapFlagsTy {
char HasNUW : 1;
char HasNSW : 1;
-
- WrapFlagsTy(bool HasNUW, bool HasNSW) : HasNUW(HasNUW), HasNSW(HasNSW) {}
+ char HasSpeculativeNSW : 1;
+
+ WrapFlagsTy(bool HasNUW, VPNoSignedWrap HasNSW)
+ : HasNUW(HasNUW), HasNSW(HasNSW == VPNoSignedWrap::ConcreteNSW),
+ HasSpeculativeNSW(HasNSW == VPNoSignedWrap::SpeculativeNSW) {}
+ WrapFlagsTy(bool HasNUW, bool HasNSW)
+ : WrapFlagsTy(HasNUW, HasNSW ? VPNoSignedWrap::ConcreteNSW
+ : VPNoSignedWrap::None) {}
WrapFlagsTy() : HasNUW(false), HasNSW(false) {}
- WrapFlagsTy withoutNoSignedWrap() {
- return {static_cast<bool>(HasNUW), false};
+ VPNoSignedWrap getNoSignedWrap() const {
+ if (HasNSW)
+ return VPNoSignedWrap::ConcreteNSW;
+ if (HasSpeculativeNSW)
+ return VPNoSignedWrap::SpeculativeNSW;
+ return VPNoSignedWrap::None;
+ }
+ bool hasNoUnsignedWrap() const { return HasNUW; }
+ WrapFlagsTy withNoUnsignedWrapIf(bool Cond) const {
+ return {hasNoUnsignedWrap() && Cond, getNoSignedWrap()};
+ }
+
+ /// Turn a concrete nsw into a speculative one, effectively dropping it
+ /// unless the speculative nsw is turned back into a concrete nsw.
+ WrapFlagsTy toSpeculativeNoSignedWrap() const {
+ return {hasNoUnsignedWrap(),
+ HasNSW ? VPNoSignedWrap::SpeculativeNSW : VPNoSignedWrap::None};
+ }
+
+ /// Turn a speculative nsw into a concrete one.
+ WrapFlagsTy fromSpeculativeNoSignedWrap() const {
+ return {hasNoUnsignedWrap(), HasSpeculativeNSW
+ ? VPNoSignedWrap::ConcreteNSW
+ : VPNoSignedWrap::None};
}
};
@@ -1020,6 +1057,7 @@ class VPIRFlags {
WrapFlagsTy getNoWrapFlagsOrNone() const {
switch (OpType) {
case OperationType::OverflowingBinOp:
+ return {WrapFlags.hasNoUnsignedWrap(), WrapFlags.getNoSignedWrap()};
case OperationType::Trunc:
return {hasNoUnsignedWrap(), hasNoSignedWrap()};
default:
@@ -1028,7 +1066,14 @@ class VPIRFlags {
}
WrapFlagsTy getNoWrapFlags() const {
- return {hasNoUnsignedWrap(), hasNoSignedWrap()};
+ switch (OpType) {
+ case OperationType::OverflowingBinOp:
+ return {WrapFlags.hasNoUnsignedWrap(), WrapFlags.getNoSignedWrap()};
+ case OperationType::Trunc:
+ return {hasNoUnsignedWrap(), hasNoSignedWrap()};
+ default:
+ llvm_unreachable("Recipe does not have wrap flags");
+ }
}
bool isDisjoint() const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 872ee86115a723..ae1b831211a7fd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -728,9 +728,8 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
if (match(BackedgeVal, m_Add(m_Specific(PhiR), m_VPValue())))
BackedgeVal->getDefiningRecipe()->setOperand(1, Step);
- // It is always safe to copy over the NoWrap and FastMath flags. In
- // particular, when folding tail by masking, the masked-off lanes are never
- // used, so it is safe.
+ // When folding tail by masking, the masked-off lanes are never used, so it is
+ // safe to use the Flags.
VPIRFlags Flags = vputils::getFlagsForInduction(IndDesc, PhiR);
auto *WideIV = new VPWidenIntOrFpInductionRecipe(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index afce807c4989c1..61c4adde5a10ae 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -60,7 +60,7 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
CanIVTy, Plan.getZero(CanIVTy), Plan.getConstantInt(CanIVTy, 1),
WideCanIV->getDebugLoc(), Builder,
- WideCanIV->getNoWrapFlags().withoutNoSignedWrap()));
+ WideCanIV->getNoWrapFlags().toSpeculativeNoSignedWrap()));
WideCanIV->eraseFromParent();
return;
}
@@ -260,7 +260,9 @@ void VPlanTransforms::materializeHeaderMask(
/// No successors
/// }
static void
-expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR) {
+expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR,
+ ElementCount VFxUF,
+ std::optional<unsigned> MaxVScale) {
VPlan *Plan = WidenIVR->getParent()->getPlan();
VPValue *Start = WidenIVR->getStartValue();
VPValue *Step = WidenIVR->getStepValue();
@@ -274,13 +276,30 @@ expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR) {
const InductionDescriptor &ID = WidenIVR->getInductionDescriptor();
Instruction::BinaryOps AddOp;
Instruction::BinaryOps MulOp;
- VPIRFlags Flags = *WidenIVR;
+ VPIRFlags Flags;
if (ID.getKind() == InductionDescriptor::IK_IntInduction) {
AddOp = Instruction::Add;
MulOp = Instruction::Mul;
+ VPIRFlags::WrapFlagsTy NWFlags = WidenIVR->getNoWrapFlags();
+
+ // Turn a speculative nsw into a concrete one.
+ const APInt *Step;
+ if (NWFlags.HasSpeculativeNSW &&
+ match(WidenIVR->getStepValue(), m_APInt(Step)) &&
+ (!VFxUF.isScalable() || MaxVScale)) {
+ bool Overflow;
+ unsigned Multiplier =
+ VFxUF.getKnownMinValue() * (VFxUF.isScalable() ? *MaxVScale : 1);
+ (void)Step->smul_ov(APInt(Step->getBitWidth(), Multiplier), Overflow);
+ if (!Overflow)
+ NWFlags = NWFlags.fromSpeculativeNoSignedWrap();
+ }
+
+ Flags = NWFlags;
} else {
AddOp = ID.getInductionOpcode();
MulOp = Instruction::FMul;
+ Flags = *WidenIVR;
}
// If the phi is truncated, truncate the start and step values.
@@ -530,7 +549,8 @@ void VPlanTransforms::expandBranchOnTwoConds(VPlan &Plan) {
}
}
-void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
+void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
+ ElementCount VFxUF) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
@@ -543,7 +563,8 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
}
if (auto *WidenIVR = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
- expandVPWidenIntOrFpInduction(WidenIVR);
+ expandVPWidenIntOrFpInduction(WidenIVR, VFxUF,
+ getMaxVScale(*Plan.getIRFunction()));
WidenIVR->eraseFromParent();
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 420c04350f8a2c..ef0aa07e28fd95 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2933,6 +2933,8 @@ void VPIRFlags::printFlags(raw_ostream &O) const {
O << " nuw";
if (WrapFlags.HasNSW)
O << " nsw";
+ if (WrapFlags.HasSpeculativeNSW)
+ O << " snsw";
break;
case OperationType::Trunc:
if (TruncFlags.HasNUW)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b6ce958a1ce1d8..2239906400759f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -885,8 +885,11 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
VPIRFlags::WrapFlagsTy WrapFlags;
// We can preserve nuw when the step is non-negative.
const APInt *Step;
- if (match(WideIV->getStepValue(), m_APInt(Step)) && Step->isNonNegative())
- WrapFlags = WideIV->getNoWrapFlagsOrNone().withoutNoSignedWrap();
+ WrapFlags = WideIV->getNoWrapFlagsOrNone()
+ .toSpeculativeNoSignedWrap()
+ .withNoUnsignedWrapIf(
+ match(WideIV->getStepValue(), m_APInt(Step)) &&
+ Step->isNonNegative());
VPScalarIVStepsRecipe *Steps = vputils::createScalarIVSteps(
Plan, ID.getKind(), ID.getInductionOpcode(),
dyn_cast_or_null<FPMathOperator>(ID.getInductionBinOp()),
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 3c793df2bf5324..9d94b93b8881c8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -422,7 +422,7 @@ struct VPlanTransforms {
static void convertToVariableLengthStep(VPlan &Plan);
/// Lower abstract recipes to concrete ones, that can be codegen'd.
- static void convertToConcreteRecipes(VPlan &Plan);
+ static void convertToConcreteRecipes(VPlan &Plan, ElementCount VFxUF);
/// This function converts initial recipes to the abstract recipes and clamps
/// \p Range based on cost model for following optimizations and cost
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 3005ef67e1f0bc..e7c486c920cf4c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -751,22 +751,21 @@ VPBasicBlock *VPBlockUtils::getPlainCFGMiddleBlock(const VPlan &Plan) {
VPIRFlags vputils::getFlagsForInduction(const InductionDescriptor &ID,
const VPPhi *PhiR) {
+ // It is always safe to use the FastMath flags from the induction descriptor.
if (ID.getKind() == InductionDescriptor::IK_FpInduction)
return ID.getInductionBinOp()->getFastMathFlags();
- // The flags only bound the induction values if the increment directly
- // updates PhiR.
+ // Get no-wrap flags for the induction from the induction increment.
VPValue *Inc = PhiR->getOperand(1);
- if (match(Inc, m_c_Add(m_Specific(PhiR), m_VPValue())))
- return cast<VPInstruction>(Inc)->getNoWrapFlagsOrNone();
-
- if (match(Inc, m_Sub(m_Specific(PhiR), m_VPValue()))) {
+ if (match(Inc, m_CombineOr(m_c_Add(m_Specific(PhiR), m_VPValue()),
+ m_Sub(m_Specific(PhiR), m_VPValue())))) {
// The step of a sub induction is negated, so NUW cannot be preserved. NSW
- // can, if the step is not the signed minimum.
- ConstantInt *Step = ID.getConstIntStepValue();
- bool NSW = cast<VPInstruction>(Inc)->getNoWrapFlagsOrNone().HasNSW &&
- Step && !Step->isMinValue(/*IsSigned=*/true);
- return VPIRFlags::WrapFlagsTy(/*NUW*/ false, NSW);
+ // can only be speculated, and concretized late when we have VF and UF
+ // information.
+ return cast<VPInstruction>(Inc)
+ ->getNoWrapFlagsOrNone()
+ .toSpeculativeNoSignedWrap()
+ .withNoUnsignedWrapIf(match(Inc, m_VPInstruction<Instruction::Add>()));
}
return VPIRFlags::WrapFlagsTy(false, false);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
index d3539eaa9e3800..6e2a3955bdaa58 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
@@ -83,7 +83,7 @@ define void @vf_dependent_uniform(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n)
; CHECK-NEXT: [[TMP37:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP36]], ptr align 8 [[TMP37]], <4 x i1> [[MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
index 7508461d4e4ec7..e295837b19a2a9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
@@ -11,7 +11,7 @@ target triple = "aarch64-none-elf"
define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK-LABEL: 'fmaxnum_reduction_f32'
-; CHECK: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 2: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to float
; CHECK: Cost of 1 for VF 2: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
@@ -42,7 +42,7 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP13]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP14]]>
; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK: Cost of 1 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK: Cost of 1 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 4: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to float
; CHECK: Cost of 1 for VF 4: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
@@ -94,7 +94,7 @@ exit:
define double @fmaxnum_reduction_f64(double %base, i64 %n) {
; CHECK-LABEL: 'fmaxnum_reduction_f64'
-; CHECK: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 2: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to double
; CHECK: Cost of 1 for VF 2: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
@@ -185,8 +185,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP34:%[0-9]+]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP35:%[0-9]+]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP36:%[0-9]+]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.4> = add ir<%c>, ir<4>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.1> = add ir<%c>, ir<1>
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 1085102596613472255 (5.882%, estimated))
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 16276538884777574400 (88.24%, estimated))
; CHECK: Cost of 7 for VF 2: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP37:%[0-9]+]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -243,8 +243,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.4> = add ir<%c>, ir<4>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.1> = add ir<%c>, ir<1>
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 1085102596613472255 (5.882%, estimated))
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 16276538884777574400 (88.24%, estimated))
; CHECK: Cost of 13 for VF 4: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -301,8 +301,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.4> = add ir<%c>, ir<4>
-; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.1> = add ir<%c>, ir<1>
+; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 1085102596613472255 (5.882%, estimated))
+; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 16276538884777574400 (88.24%, estimated))
; CHECK: Cost of 10 for VF 8: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 8: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 8: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -359,8 +359,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.4> = add ir<%c>, ir<4>
-; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.1> = add ir<%c>, ir<1>
+; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 1085102596613472255 (5.882%, estimated))
+; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 16276538884777574400 (88.24%, estimated))
; CHECK: Cost of 20 for VF 16: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 16: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 16: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -442,7 +442,7 @@ define void @blend_fcmp_olt_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%c> = fcmp olt ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK: Cost of 1 for VF 2: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
@@ -465,7 +465,7 @@ define void @blend_fcmp_olt_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%c> = fcmp olt ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK: Cost of 1 for VF 4: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
@@ -518,7 +518,7 @@ define void @blend_fcmp_uno_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 3 for VF 2: WIDEN ir<%c> = fcmp uno ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 17592186044415 (9.537E-5%, estimated))
; CHECK: Cost of 2 for VF 2: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
@@ -541,7 +541,7 @@ define void @blend_fcmp_uno_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 3 for VF 4: WIDEN ir<%c> = fcmp uno ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 17592186044415 (9.537E-5%, estimated))
; CHECK: Cost of 2 for VF 4: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
index ecb7fff77c9605..5b823bdf42cfd0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
@@ -414,7 +414,7 @@ define i32 @multi_use_cmp_for_csa_int_select(i64 %N, ptr %data, i32 %a) {
; SVE-NEXT: [[TMP11]] = select i1 [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
; SVE-NEXT: [[TMP12]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i64> [[VEC_IND]], <vscale x 4 x i64> [[VEC_PHI3]]
; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
-; SVE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; SVE-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; SVE-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; SVE-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; SVE: [[MIDDLE_BLOCK]]:
@@ -1276,7 +1276,7 @@ define i32 @csa_load_nested_ifs(ptr noalias %a, ptr noalias %b, i32 %default_val
; SVE-NEXT: [[TMP13]] = select i1 [[TMP12]], <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> [[TMP5]]
; SVE-NEXT: [[TMP14]] = select i1 [[TMP12]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD5]], <vscale x 4 x i32> [[VEC_PHI]]
; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; SVE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; SVE-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; SVE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; SVE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; SVE: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
index 7facdb205a81ef..8ff2dc8f94aae1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
@@ -127,4 +127,4 @@ exit:
ret void
}
-attributes #0 = { "target-features"="+sve" optsize }
+attributes #0 = { "target-features"="+sve" vscale_range(2, 1024) optsize }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
index 2e858bdcaddc58..c4f701917bd639 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
@@ -24,7 +24,7 @@ define i8 @select_icmp_var_start(ptr %a, i8 %n, i8 %start) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <16 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i8> [ splat (i8 -128), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i8> [ splat (i8 -128), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <16 x i8> [[VEC_IND]], splat (i8 16)
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <16 x i8> [[VEC_IND]], splat (i8 16)
; CHECK-NEXT: [[INDEX4:%.*]] = trunc i32 [[INDEX]] to i8
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i8 [[INDEX4]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 16
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
index 5a2a403174f633..e52751db2fa05a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
@@ -204,7 +204,7 @@ define void @test_widen_induction_variable_start(ptr %A, i64 %N, i64 %start) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add nuw i64 [[START]], [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i64 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
index 0544f52778b560..c81342fa18f710 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
@@ -103,7 +103,7 @@ for.body:
define i1 @test_extra_cmp_user(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src) {
; CHECK-LABEL: LV: Checking a loop in 'test_extra_cmp_user'
-; CHECK: Cost of 4 for VF 8: ir<%indvars.iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<{{.*}}>
+; CHECK: Cost of 4 for VF 8: ir<%indvars.iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<{{.*}}>
; CHECK: Cost of 4 for VF 8: WIDEN ir<%exitcond.not> = icmp eq ir<%indvars.iv.next>, ir<16>
; CHECK: Cost of 1 for VF 8: EMIT vp<%cmp.n> = icmp eq ir<16>, vp<%2>
; CHECK-NEXT: Cost of 0 for VF 8: EMIT branch-on-cond vp<%cmp.n>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
index ab198def82fa38..be2b43a7fc0fd8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
@@ -97,9 +97,9 @@ exit:
; The lanes of the column-major store A[i + j*8] are only distinct for VF <= 8,
; which a scalable VF cannot guarantee.
-define void @col_major_m8(ptr noalias %A, i64 %N) {
+define void @col_major_m8(ptr noalias %A, i64 %N) vscale_range(2, 1024) {
; CHECK-LABEL: define void @col_major_m8(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
index a7a2e040cbe52b..7d3db2a24b7c91 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
@@ -173,7 +173,7 @@ define void @test_invar_gep_var_start(i64 %start, ptr %dst) #0 {
; IC2: vector.body:
; IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; IC2-NEXT: [[DOTSPLAT:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IC2-NEXT: [[TMP10:%.*]] = add nsw <vscale x 4 x i64> [[DOTSPLAT]], [[TMP9]]
+; IC2-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[DOTSPLAT]], [[TMP9]]
; IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; IC2-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[TMP10]], [[TMP9]]
; IC2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -293,7 +293,7 @@ define void @test_invar_gep_var_start_step_2(i64 %start, ptr %dst) #0 {
; IC2: vector.body:
; IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; IC2-NEXT: [[DOTSPLAT:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IC2-NEXT: [[TMP14:%.*]] = add nsw <vscale x 4 x i64> [[DOTSPLAT]], [[TMP13]]
+; IC2-NEXT: [[TMP14:%.*]] = add <vscale x 4 x i64> [[DOTSPLAT]], [[TMP13]]
; IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
; IC2-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[TMP14]], [[TMP13]]
; IC2-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-induction-wrapflags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-induction-wrapflags.ll
index f39516c3a327d0..2161ad5210c01a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-induction-wrapflags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-induction-wrapflags.ll
@@ -18,9 +18,9 @@ define void @i8_induction_vf_may_exceed_signed_max(ptr noalias %dst, i64 %n) #0
; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[N_VEC]] to i8
; CHECK-NEXT: [[TMP3:%.*]] = sub i8 0, [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
-; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <vscale x 8 x i8> zeroinitializer, [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = sub <vscale x 8 x i8> zeroinitializer, [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP1]] to i8
-; CHECK-NEXT: [[TMP7:%.*]] = sub nsw i8 0, [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = sub i8 0, [[TMP6]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[TMP7]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -30,7 +30,7 @@ define void @i8_induction_vf_may_exceed_signed_max(ptr noalias %dst, i64 %n) #0
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <vscale x 8 x i8> [[VEC_IND]], ptr [[TMP8]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 8 x i8> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i8> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -69,9 +69,9 @@ define void @i8_induction_unknown_max_vscale(ptr noalias %dst, i64 %n) #1 {
; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[N_VEC]] to i8
; CHECK-NEXT: [[TMP3:%.*]] = sub i8 0, [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
-; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <vscale x 8 x i8> zeroinitializer, [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = sub <vscale x 8 x i8> zeroinitializer, [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP1]] to i8
-; CHECK-NEXT: [[TMP7:%.*]] = sub nsw i8 0, [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = sub i8 0, [[TMP6]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[TMP7]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -81,7 +81,7 @@ define void @i8_induction_unknown_max_vscale(ptr noalias %dst, i64 %n) #1 {
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <vscale x 8 x i8> [[VEC_IND]], ptr [[TMP8]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 8 x i8> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i8> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll
index 1dc8d6c8049799..9e9afa9a5f50a5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll
@@ -35,7 +35,7 @@ define void @cond_ind64(ptr noalias nocapture %a, ptr noalias nocapture readonly
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP11]], <vscale x 4 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
index e67370e37f92e8..3c8d7ae90dc56b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
@@ -21,7 +21,7 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
; }
; }
;}
-define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) #0 {
+define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) vscale_range(2, 1024) {
; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided1(
; SCALAR_TAIL_FOLDING-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] {
; SCALAR_TAIL_FOLDING-NEXT: [[ENTRY:.*:]]
@@ -75,7 +75,6 @@ define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias noca
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
-; PREDICATED_TAIL_FOLDING-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP1]])
; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
@@ -102,11 +101,11 @@ define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias noca
; PREDICATED_TAIL_FOLDING-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP12]])
; PREDICATED_TAIL_FOLDING-NEXT: [[INTERLEAVED_MASK3:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP11]], <vscale x 32 x i1> [[INTERLEAVED_MASK3]])
-; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TRIP_COUNT_MINUS_VF]])
+; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1024)
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; PREDICATED_TAIL_FOLDING: [[MIDDLE_BLOCK]]:
; PREDICATED_TAIL_FOLDING-NEXT: br label %[[FOR_END:.*]]
@@ -160,7 +159,7 @@ for.end:
; q[2*ix+1] = 2;
; }
;}
-define void @masked_strided2(ptr noalias nocapture readnone %p, ptr noalias nocapture %q, i8 zeroext %guard) #0 {
+define void @masked_strided2(ptr noalias nocapture readnone %p, ptr noalias nocapture %q, i8 zeroext %guard) vscale_range(2, 1024) {
; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided2(
; SCALAR_TAIL_FOLDING-SAME: ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
; SCALAR_TAIL_FOLDING-NEXT: [[ENTRY:.*:]]
@@ -206,7 +205,6 @@ define void @masked_strided2(ptr noalias nocapture readnone %p, ptr noalias noca
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
-; PREDICATED_TAIL_FOLDING-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP1]])
; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
@@ -225,11 +223,11 @@ define void @masked_strided2(ptr noalias nocapture readnone %p, ptr noalias noca
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = or disjoint <vscale x 16 x i32> [[TMP3]], splat (i32 1)
; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i32> [[TMP6]]
; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 2), <vscale x 16 x ptr> align 1 [[WIDE_GEP3]], <vscale x 16 x i1> [[TMP5]])
-; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TRIP_COUNT_MINUS_VF]])
+; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1024)
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = xor i1 [[TMP7]], true
; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; PREDICATED_TAIL_FOLDING: [[MIDDLE_BLOCK]]:
; PREDICATED_TAIL_FOLDING-NEXT: br label %[[FOR_END:.*]]
@@ -278,7 +276,7 @@ for.end:
; }
; }
;}
-define void @masked_strided3(ptr noalias nocapture readnone %p, ptr noalias nocapture %q, i8 zeroext %guard1, i8 zeroext %guard2) #0 {
+define void @masked_strided3(ptr noalias nocapture readnone %p, ptr noalias nocapture %q, i8 zeroext %guard1, i8 zeroext %guard2) vscale_range(2, 1024) {
; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided3(
; SCALAR_TAIL_FOLDING-SAME: ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD1:%.*]], i8 zeroext [[GUARD2:%.*]]) #[[ATTR0]] {
; SCALAR_TAIL_FOLDING-NEXT: [[ENTRY:.*:]]
@@ -329,7 +327,6 @@ define void @masked_strided3(ptr noalias nocapture readnone %p, ptr noalias noca
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
-; PREDICATED_TAIL_FOLDING-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP1]])
; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV3]], i64 0
@@ -352,11 +349,11 @@ define void @masked_strided3(ptr noalias nocapture readnone %p, ptr noalias noca
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = or disjoint <vscale x 16 x i32> [[TMP3]], splat (i32 1)
; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_GEP5:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i32> [[TMP8]]
; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 2), <vscale x 16 x ptr> align 1 [[WIDE_GEP5]], <vscale x 16 x i1> [[TMP7]])
-; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TRIP_COUNT_MINUS_VF]])
+; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1024)
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP9:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
-; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; PREDICATED_TAIL_FOLDING: [[MIDDLE_BLOCK]]:
; PREDICATED_TAIL_FOLDING-NEXT: br label %[[FOR_END:.*]]
@@ -418,7 +415,7 @@ for.end:
; }
; }
;}
-define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) #0 {
+define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) vscale_range(2, 1024) {
; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided_factor4(
; SCALAR_TAIL_FOLDING-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
; SCALAR_TAIL_FOLDING-NEXT: [[ENTRY:.*:]]
@@ -477,7 +474,6 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
-; PREDICATED_TAIL_FOLDING-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP1]])
; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
@@ -509,11 +505,11 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
; PREDICATED_TAIL_FOLDING-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP12]], <vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]])
; PREDICATED_TAIL_FOLDING-NEXT: [[INTERLEAVED_MASK3:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP17]], <vscale x 64 x i1> [[INTERLEAVED_MASK3]])
-; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TRIP_COUNT_MINUS_VF]])
+; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1024)
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP18:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = xor i1 [[TMP18]], true
; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; PREDICATED_TAIL_FOLDING: [[MIDDLE_BLOCK]]:
; PREDICATED_TAIL_FOLDING-NEXT: br label %[[FOR_END:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
index f27ebb085dbec0..b3941a83bafb6c 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
@@ -450,9 +450,9 @@ define void @test_stride_noninvar3_4i32(ptr readonly %data, ptr noalias nocaptur
; CHECK-NEXT: [[IND_END:%.*]] = add i32 3, [[TMP0]]
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = mul nuw nsw <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[DOTSPLAT]]
-; CHECK-NEXT: [[INDUCTION:%.*]] = add nuw nsw <4 x i32> splat (i32 3), [[TMP3]]
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i32 [[X]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = mul nuw <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[DOTSPLAT]]
+; CHECK-NEXT: [[INDUCTION:%.*]] = add nuw <4 x i32> splat (i32 3), [[TMP3]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i32 [[X]], 2
; CHECK-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i64 0
; CHECK-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -469,7 +469,7 @@ define void @test_stride_noninvar3_4i32(ptr readonly %data, ptr noalias nocaptur
; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
-; CHECK-NEXT: [[VEC_IND_NEXT5]] = add nuw nsw <4 x i32> [[VEC_IND4]], [[DOTSPLAT3]]
+; CHECK-NEXT: [[VEC_IND_NEXT5]] = add nuw <4 x i32> [[VEC_IND4]], [[DOTSPLAT3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dbg-tail-folding-by-evl.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dbg-tail-folding-by-evl.ll
index e7405906b16685..01bb5d9b6469af 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dbg-tail-folding-by-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dbg-tail-folding-by-evl.ll
@@ -15,14 +15,14 @@ define void @reverse_store(ptr %a, i64 %n) !dbg !3 {
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[N]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = sub nsw <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP2]]
+; CHECK-NEXT: [[TMP15:%.*]] = sub <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP2]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP15]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP9]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
-; CHECK-NEXT: [[TMP6:%.*]] = sub nsw i64 0, [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP5]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP6]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 2 x i64> [[VEC_IND]], splat (i64 -1), !dbg [[DBG6:![0-9]+]]
@@ -34,7 +34,7 @@ define void @reverse_store(ptr %a, i64 %n) !dbg !3 {
; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.reverse.nxv2i64(<vscale x 2 x i64> [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP4]]), !dbg [[DBG8]]
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP14]], ptr align 8 [[TMP13]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP4]]), !dbg [[DBG8]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll b/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
index cae9837e84c822..49e5dadb8d5565 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
@@ -4,7 +4,7 @@
; RUN: -mtriple=riscv64 -mattr=+v -S < %s | FileCheck %s
; Make sure we do not vectorize a loop with a widened int induction.
-define void @test_wide_integer_induction(ptr noalias %a, i64 %N) {
+define void @test_wide_integer_induction(ptr noalias %a, i64 %N) vscale_range(2, 1024) {
; CHECK-LABEL: define void @test_wide_integer_induction(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -50,7 +50,7 @@ for.cond.cleanup:
; Make sure we do not vectorize a loop with a widened ptr induction.
define void @test_wide_ptr_induction(ptr noalias %a, ptr noalias %b, i64 %N) {
; CHECK-LABEL: define void @test_wide_ptr_induction(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll
index 1306b9ae63807f..64fe26f293fb06 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll
@@ -58,9 +58,9 @@ exit:
ret i32 %rdx.next
}
-define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64 %rdx.start, i64 %n) {
+define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64 %rdx.start, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: define i64 @select_decreasing_induction_icmp_non_const_start(
-; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[N]], 1
@@ -70,11 +70,11 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[N]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = sub nsw <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP2]]
+; CHECK-NEXT: [[TMP13:%.*]] = sub nsw <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP2]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP8]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
index 5e5c39396c9fb5..9375dc7fdb630d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
@@ -1162,9 +1162,9 @@ exit:
ret void
}
-define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
+define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @vp_ptrtoint(
-; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
; IF-EVL-NEXT: [[ENTRY:.*:]]
; IF-EVL-NEXT: br label %[[VECTOR_PH:.*]]
; IF-EVL: [[VECTOR_PH]]:
@@ -1193,12 +1193,11 @@ define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
; IF-EVL-NEXT: ret void
;
; NO-VP-LABEL: define void @vp_ptrtoint(
-; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
; NO-VP-NEXT: [[ENTRY:.*]]:
; NO-VP-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP13]], 1
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 4)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll
index cebcece2d13406..6a6af2a10f7ad4 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll
@@ -505,9 +505,9 @@ for.end:
ret i32 %for1
}
-define void @first_order_recurrence_indvar(ptr noalias %A, i64 %TC) {
+define void @first_order_recurrence_indvar(ptr noalias %A, i64 %TC) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @first_order_recurrence_indvar(
-; IF-EVL-SAME: ptr noalias [[A:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
+; IF-EVL-SAME: ptr noalias [[A:%.*]], i64 [[TC:%.*]]) #[[ATTR1:[0-9]+]] {
; IF-EVL-NEXT: [[ENTRY:.*:]]
; IF-EVL-NEXT: br label %[[VECTOR_PH:.*]]
; IF-EVL: [[VECTOR_PH]]:
@@ -545,7 +545,7 @@ define void @first_order_recurrence_indvar(ptr noalias %A, i64 %TC) {
; IF-EVL-NEXT: ret void
;
; NO-VP-LABEL: define void @first_order_recurrence_indvar(
-; NO-VP-SAME: ptr noalias [[A:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
+; NO-VP-SAME: ptr noalias [[A:%.*]], i64 [[TC:%.*]]) #[[ATTR1:[0-9]+]] {
; NO-VP-NEXT: [[ENTRY:.*]]:
; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
index 0b2895c01108b5..83e3985429f99f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
@@ -1665,7 +1665,7 @@ for.end:
; Test for multiuse-reductions that are tailfolded using strict predicates.
-define i64 @argmin_first_index(ptr %arr, i64 %n, i64 %start) {
+define i64 @argmin_first_index(ptr %arr, i64 %n, i64 %start) vscale_range(2, 1024) {
; IF-EVL-LABEL: @argmin_first_index(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
@@ -1790,7 +1790,7 @@ for.end:
}
-define i64 @argmin_last_index(ptr %arr, i64 %n, i64 %start) {
+define i64 @argmin_last_index(ptr %arr, i64 %n, i64 %start) vscale_range(2, 1024) {
; IF-EVL-LABEL: @argmin_last_index(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
@@ -1915,7 +1915,7 @@ for.end:
}
-define i64 @argmax_first_index(ptr %arr, i64 %n, i64 %start) {
+define i64 @argmax_first_index(ptr %arr, i64 %n, i64 %start) vscale_range(2, 1024) {
; IF-EVL-LABEL: @argmax_first_index(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
@@ -2040,7 +2040,7 @@ for.end:
}
-define i64 @argmax_last_index(ptr %arr, i64 %n, i64 %start) {
+define i64 @argmax_last_index(ptr %arr, i64 %n, i64 %start) vscale_range(2, 1024) {
; IF-EVL-LABEL: @argmax_last_index(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
index f141aa0a6420ca..7a42413a731a9a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
@@ -23,7 +23,7 @@ define void @pick_wide_call(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds double, ir<%in.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
@@ -77,7 +77,7 @@ define void @pick_intrinsic_call(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i64, ir<%in.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/scalarize-after-call-widening.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/scalarize-after-call-widening.ll
index fef84c5def4863..733dfb8bb35261 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/scalarize-after-call-widening.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/scalarize-after-call-widening.ll
@@ -16,7 +16,7 @@ define void @uniform_arg(ptr noalias %src, ptr noalias %dst, i64 %a, i64 %b) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i64, ir<%gep.src>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP4]]>
@@ -58,7 +58,7 @@ define void @linear_arg(ptr noalias %dst) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK-NEXT: CLONE ir<%iv.mul> = mul ir<%iv>, ir<3>
; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo_linear(ir<%iv.mul>) (using library function: foo_linear_vec)
; CHECK-NEXT: CLONE ir<%gep.dst> = getelementptr inbounds ir<%dst>, ir<%iv>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
index f9a3205e89a7c2..deda6cc8fcf780 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
@@ -83,8 +83,8 @@ define void @load_feeding_only_mask_not_scalarized(ptr noalias %A, ptr noalias %
; CHECK-NEXT: then:
; CHECK-NEXT: EMIT ir<%gep.B> = getelementptr ir<%B>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ptr, ir<%gep.B>, ir<1>
-; CHECK-NEXT: WIDEN ir<%l.p> = load vp<[[VP5]]>, ir<%cmp>
-; CHECK-NEXT: EMIT store ir<42>, ir<%l.p>, ir<%cmp>
+; CHECK-NEXT: WIDEN ir<%l.p> = load vp<[[VP5]]>, ir<%cmp> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
+; CHECK-NEXT: EMIT store ir<42>, ir<%l.p>, ir<%cmp> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
@@ -196,7 +196,7 @@ define void @consecutive_load_store(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.a>, ir<1>
; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP4]]>
@@ -251,7 +251,7 @@ define void @reverse_load_store(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<1023>, ir<-1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<1023>, ir<-1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-end-pointer inbounds i32, ir<%gep.a>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP4]]>
@@ -308,7 +308,7 @@ define void @consecutive_load_with_first_order_recurrence_address(ptr noalias %a
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: ir<%narrow> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%prev> = phi ir<0>, ir<%ext>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
@@ -369,7 +369,7 @@ define void @cond_load_store(ptr noalias %a, ptr noalias %b, ptr noalias %cond,
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%gep.cond> = getelementptr inbounds ir<%cond>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.cond>, ir<1>
; CHECK-NEXT: WIDEN ir<%c> = load vp<[[VP4]]>
@@ -379,11 +379,11 @@ define void @cond_load_store(ptr noalias %a, ptr noalias %b, ptr noalias %cond,
; CHECK-NEXT: then:
; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.a>, ir<1>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP5]]>, ir<%cmp>
-; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>, ir<%cmp>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP5]]>, ir<%cmp> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
+; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>, ir<%cmp> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.b>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%add>, ir<%cmp>
+; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%add>, ir<%cmp> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
@@ -444,7 +444,7 @@ define void @cond_reverse_load_store(ptr noalias %a, ptr noalias %b, ptr noalias
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<1023>, ir<-1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<1023>, ir<-1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%gep.cond> = getelementptr inbounds ir<%cond>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-end-pointer inbounds i32, ir<%gep.cond>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN ir<%c> = load vp<[[VP4]]>
@@ -456,14 +456,14 @@ define void @cond_reverse_load_store(ptr noalias %a, ptr noalias %b, ptr noalias
; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-end-pointer inbounds i32, ir<%gep.a>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = reverse ir<%cmp>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP6]]>, vp<[[VP7]]> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = reverse ir<%lv>
-; CHECK-NEXT: EMIT ir<%add> = add vp<[[VP8]]>, ir<1>, ir<%cmp>
+; CHECK-NEXT: EMIT ir<%add> = add vp<[[VP8]]>, ir<1>, ir<%cmp> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-end-pointer inbounds i32, ir<%gep.b>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = reverse ir<%cmp>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = reverse ir<%add>
-; CHECK-NEXT: WIDEN store vp<[[VP9]]>, vp<[[VP11]]>, vp<[[VP10]]>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, vp<[[VP11]]>, vp<[[VP10]]> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
@@ -523,7 +523,7 @@ define void @or_disjoint_address(ptr noalias %A) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%or> = or disjoint ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%or>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep>, ir<1>
@@ -579,7 +579,7 @@ define void @sdiv_with_nonnegative_operands_address(ptr noalias %A) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%mul> = mul nuw nsw ir<%iv>, ir<4>
; CHECK-NEXT: EMIT ir<%div> = sdiv ir<%mul>, ir<4>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%div>
@@ -637,7 +637,7 @@ define void @ashr_of_shl_address(ptr noalias %A) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%shl> = shl ir<%iv>, ir<32>
; CHECK-NEXT: EMIT ir<%ashr> = ashr ir<%shl>, ir<32>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%ashr>
@@ -694,7 +694,7 @@ define void @extractvalue_of_with_overflow_address(ptr noalias %A) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%res> = call { i64, i1 } @llvm.sadd.with.overflow.i64(ir<%iv>, ir<1>)
; CHECK-NEXT: EMIT ir<%idx> = extractvalue ir<%res>, ir<0>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
@@ -749,16 +749,16 @@ define void @blend_with_identical_incoming_values_address(ptr noalias %A, i1 %c)
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: Successor(s): else
; CHECK-EMPTY:
; CHECK-NEXT: else:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c>
-; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): then
; CHECK-EMPTY:
; CHECK-NEXT: then:
-; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>, ir<%c>
+; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>, ir<%c> (!vplan.execution.frequency 9223372036854775808 (50%, estimated))
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
@@ -826,7 +826,7 @@ define void @or_without_disjoint_address(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%idx> = or ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.B> = getelementptr inbounds ir<%B>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.B>
@@ -880,7 +880,7 @@ define void @sdiv_with_negative_operand_address(ptr noalias %A, ptr noalias %B)
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%mul> = mul nuw nsw ir<%iv>, ir<4>
; CHECK-NEXT: EMIT ir<%idx> = sdiv ir<%mul>, ir<-4>
; CHECK-NEXT: EMIT ir<%gep.B> = getelementptr inbounds ir<%B>, ir<%idx>
@@ -936,7 +936,7 @@ define void @ashr_amount_larger_than_shl_address(ptr noalias %A, ptr noalias %B)
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%shl> = shl ir<%iv>, ir<16>
; CHECK-NEXT: EMIT ir<%idx> = ashr ir<%shl>, ir<32>
; CHECK-NEXT: EMIT ir<%gep.B> = getelementptr inbounds ir<%B>, ir<%idx>
@@ -992,16 +992,16 @@ define void @blend_with_different_incoming_values_address(ptr noalias %A, ptr no
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: Successor(s): else
; CHECK-EMPTY:
; CHECK-NEXT: else:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c>
-; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-2>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): then
; CHECK-EMPTY:
; CHECK-NEXT: then:
-; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>, ir<%c>
+; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>, ir<%c> (!vplan.execution.frequency 9223372036854775808 (50%, estimated))
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
index 7bab46500720a4..4971992f4b3a6b 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
@@ -203,7 +203,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) vscale_range(2, 1024) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP4:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP5:%[0-9]+]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP5:%[0-9]+]]>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-iv) ir<-9223372036854775808>, vp<[[VP8:%[0-9]+]]>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index bc814cf618c6e9..0cd8886aa8b6aa 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -21,7 +21,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
@@ -110,7 +110,7 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
@@ -195,7 +195,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
@@ -287,7 +287,7 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ee.ptr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i32, ir<%ee.ptr>, ir<1>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll b/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll
index d7188f4ffd59de..bfdee7b91a3a01 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll
@@ -19,7 +19,7 @@ define i32 @find_last_with_select(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-last) ir<1>, ir<%sel>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.a>, ir<1>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
index 313dda16a48bc1..fcb108f96b7d59 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
@@ -140,7 +140,7 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr, i32 %z) optsize {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = first-order splice ir<%recur>, ir<%recur.next>
@@ -150,14 +150,14 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr, i32 %z) optsize {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[VP8]]>, ir<%x>
; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP10]]>
-; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next>
+; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll b/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
index af9dbf0fa36b70..b931957d09339a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
@@ -20,7 +20,7 @@ define i32 @test(i32 %limit, i1 %guard, i32 %divisor) {
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll
index fd075a9f72419d..82cf14c9afb0f0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll
@@ -8,7 +8,7 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
; recognized as uniform and is vectorized.
;
define i32 @more_than_one_use(ptr %a, i64 %n) {
-; CHECK-LABEL: VPlan for loop in 'more_than_one_use' after printOptimizedVPlan
+; CHECK-LABEL: VPlan for loop in 'more_than_one_use'
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
@@ -27,7 +27,7 @@ define i32 @more_than_one_use(ptr %a, i64 %n) {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%i> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%i> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%r> = phi (add) vp<[[VP4]]>, ir<%tmp3>
; CHECK-NEXT: WIDEN ir<%i.next> = add nuw nsw ir<%i>, ir<1>
; CHECK-NEXT: WIDEN ir<%cond> = icmp sge ir<%i.next>, ir<%n>
@@ -106,7 +106,7 @@ define void @test(ptr %ptr) {
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
; CHECK-NEXT: WIDEN ir<%cond0> = icmp ult ir<%iv>, ir<13>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 483ab8ad94b50a..a172389135a177 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -7,18 +7,18 @@ define void @diamond_phi(ptr %a) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%c0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%c0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated))
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
@@ -72,26 +72,26 @@ define void @mask_reuse(ptr %a) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%c0>
-; CHECK-NEXT: EMIT ir<%c1> = icmp sle ir<%iv>, ir<1>, ir<%c0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%c0> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
+; CHECK-NEXT: EMIT ir<%c1> = icmp sle ir<%iv>, ir<1>, ir<%c0> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]> (!vplan.execution.frequency 4611686018427387903 (25%, estimated))
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]>
; CHECK-NEXT: BLEND ir<%phi3> = ir<%add2>/vp<[[VP4]]> ir<%add1>/vp<[[VP6]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, ir<%c0>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, ir<%c0> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
@@ -157,38 +157,38 @@ define void @optimized_mask(ptr %a) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): bb6
; CHECK-EMPTY:
; CHECK-NEXT: bb6:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
-; CHECK-NEXT: EMIT ir<%add6> = add ir<%iv>, ir<6>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT ir<%c6> = icmp sle ir<%iv>, ir<6>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add6> = add ir<%iv>, ir<6>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
+; CHECK-NEXT: EMIT ir<%c6> = icmp sle ir<%iv>, ir<6>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT ir<%c1> = icmp sle ir<%iv>, ir<1>, ir<%c0>
+; CHECK-NEXT: EMIT ir<%c1> = icmp sle ir<%iv>, ir<1>, ir<%c0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated))
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT ir<%c3> = icmp sle ir<%iv>, ir<3>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated))
+; CHECK-NEXT: EMIT ir<%c3> = icmp sle ir<%iv>, ir<3>, vp<[[VP6]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated))
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP7]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated))
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%c3>
; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = or vp<[[VP8]]>, vp<[[VP7]]>
; CHECK-NEXT: BLEND ir<%phi4> = ir<%add3>/vp<[[VP8]]> ir<%add2>/vp<[[VP7]]>
-; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP9]]> (!vplan.execution.frequency 6917529027641081856 (37.5%, estimated))
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
@@ -198,7 +198,7 @@ define void @optimized_mask(ptr %a) {
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = logical-and vp<[[VP6]]>, vp<[[VP12]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = or vp<[[VP11]]>, vp<[[VP13]]>
; CHECK-NEXT: BLEND ir<%phi5> = ir<%add6>/vp<[[VP10]]> ir<%add4>/vp<[[VP9]]> ir<%add3>/vp<[[VP13]]>
-; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, vp<[[VP14]]>
+; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, vp<[[VP14]]> (!vplan.execution.frequency 13835058055282163711 (75%, estimated))
; CHECK-NEXT: Successor(s): bb7
; CHECK-EMPTY:
; CHECK-NEXT: bb7:
@@ -282,19 +282,19 @@ define void @switch(ptr %a) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT ir<%c2> = icmp sle ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
+; CHECK-NEXT: EMIT ir<%c2> = icmp sle ir<%iv>, ir<2>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%c0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%c0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated))
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
@@ -310,11 +310,11 @@ define void @switch(ptr %a) {
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP13]]>
; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP11]]>
; CHECK-NEXT: BLEND ir<%phi3> = ir<%add2>/vp<[[VP5]]> ir<%add1>/vp<[[VP11]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP15]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP15]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP9]]> (!vplan.execution.frequency 2305843009213693952 (12.5%, estimated))
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
@@ -399,17 +399,17 @@ define void @diamond_phi2(ptr %a, i1 %c1, i1 %c2) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%c0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%c0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated))
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
@@ -418,7 +418,7 @@ define void @diamond_phi2(ptr %a, i1 %c1, i1 %c2) {
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
; CHECK-NEXT: BLEND ir<%phi> = ir<%add2>/vp<[[VP5]]> ir<%add1>/vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP7]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
@@ -519,7 +519,7 @@ define void @blend_masks(ptr noalias %p, i1 %c0, i1 %c1, i1 %c2, i1 %c3, i1 %c4)
; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = or vp<[[VP15]]>, vp<[[VP14]]>
; CHECK-NEXT: BLEND ir<%phi> = ir<1>/vp<[[VP15]]> ir<0>/vp<[[VP14]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP16]]>
+; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP16]]> (!vplan.execution.frequency 10376293541461622783 (56.25%, estimated))
; CHECK-NEXT: Successor(s): bb8
; CHECK-EMPTY:
; CHECK-NEXT: bb8:
@@ -645,7 +645,7 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK-NEXT: EMIT-SCALAR ir<%lb> = load ir<%b>
; CHECK-NEXT: EMIT ir<%v1> = add ir<%iv>, ir<%lb>
; CHECK-NEXT: EMIT ir<%v2> = mul ir<%iv>, ir<3>
@@ -654,7 +654,7 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: Successor(s): if.a
; CHECK-EMPTY:
; CHECK-NEXT: if.a:
-; CHECK-NEXT: EMIT ir<%ca> = icmp sle ir<%iv>, ir<8>, ir<%c0>
+; CHECK-NEXT: EMIT ir<%ca> = icmp sle ir<%iv>, ir<8>, ir<%c0> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): if.a.inner
; CHECK-EMPTY:
; CHECK-NEXT: if.a.inner:
@@ -671,7 +671,7 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: Successor(s): if.b
; CHECK-EMPTY:
; CHECK-NEXT: if.b:
-; CHECK-NEXT: EMIT ir<%cb> = icmp sle ir<%iv>, ir<16>, ir<%d0>
+; CHECK-NEXT: EMIT ir<%cb> = icmp sle ir<%iv>, ir<16>, ir<%d0> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
; CHECK-NEXT: Successor(s): if.b.inner
; CHECK-EMPTY:
; CHECK-NEXT: if.b.inner:
@@ -861,7 +861,7 @@ define void @phi_doesnt_postdom_incoming(i1 %c1, i1 %c2, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP5]]>
; CHECK-NEXT: BLEND ir<%phi> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP5]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP6]]> (!vplan.execution.frequency 13835058055282163711 (75%, estimated))
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
@@ -923,7 +923,7 @@ define void @phi_switch_same_edge(ptr %p) {
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<0>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<0>, ir<%gep>, vp<[[VP6]]> (!vplan.execution.frequency 12297829379609722879 (66.67%, estimated))
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/print-attributes.ll b/llvm/test/Transforms/LoopVectorize/VPlan/print-attributes.ll
index 4a744ddd830fb8..e06d2bdd3e385f 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/print-attributes.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/print-attributes.ll
@@ -29,7 +29,7 @@ define void @wide_call_attrs(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds double, ir<%in.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
@@ -95,7 +95,7 @@ define void @wide_call_attrs_filtered(ptr noalias %in.ptr, ptr noalias %out.ptr)
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i64, ir<%in.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
@@ -225,7 +225,7 @@ define void @libcall_widened_as_intrinsic_call_site_fn_attrs(ptr noalias %in.ptr
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds float, ir<%in.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/print-calls.ll b/llvm/test/Transforms/LoopVectorize/VPlan/print-calls.ll
index c6e66ccc3114e8..25056151d7d23f 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/print-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/print-calls.ll
@@ -26,7 +26,7 @@ define void @call_site_attrs(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i64, ir<%in.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
@@ -81,7 +81,7 @@ define void @masked_call_site_attrs(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i64, ir<%in.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
@@ -149,7 +149,7 @@ define void @call_site_memory_attrs(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i64, ir<%in.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
@@ -204,7 +204,7 @@ define void @call_quoted_name(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i64, ir<%in.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/simplify-reverse-reverse.ll b/llvm/test/Transforms/LoopVectorize/VPlan/simplify-reverse-reverse.ll
index e836418da28f01..844c5e2e37eb08 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/simplify-reverse-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/simplify-reverse-reverse.ll
@@ -21,7 +21,7 @@ define void @reverse_copy(ptr noalias %src, ptr noalias %dst, i32 %n) {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<%n>, ir<-1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<%n>, ir<-1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<%n> + vp<[[VP5]]> * ir<-1>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<-1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%iv.next> = add nsw vp<[[VP7]]>, ir<-1>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding-multiuse-reductions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding-multiuse-reductions.ll
index f83d4cbeb14553..51a3e42847ceb1 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding-multiuse-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding-multiuse-reductions.ll
@@ -20,7 +20,7 @@ define i64 @argmin_first_index(ptr %arr, i64 %n, i64 %start) {
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%min> = phi (smin) ir<%start>, vp<[[VP6:%[0-9]+]]>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%min.loc> = phi (find-iv) ir<poison>, vp<[[VP7:%[0-9]+]]>
; CHECK-NEXT: Successor(s): vector.body.split
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
index 7f44bc7c68294b..1ce3cd841057e6 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -21,7 +21,7 @@ define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -88,7 +88,7 @@ define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -157,7 +157,7 @@ define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -230,7 +230,7 @@ define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<8>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
@@ -301,7 +301,7 @@ define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %strid
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<4>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
@@ -373,7 +373,7 @@ define void @byte_gep_under_scaled_stride_extra_narrow_use(ptr noalias %p.out, p
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<4>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
@@ -451,7 +451,7 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<16>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
@@ -522,7 +522,7 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<11>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
@@ -592,7 +592,7 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -662,7 +662,7 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%stride.neg> = sub ir<0>, ir<%stride>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.neg>
@@ -732,7 +732,7 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx>
@@ -806,7 +806,7 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride0>
; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<%stride1>
@@ -882,7 +882,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%stride1> = add ir<%stride>, ir<1>
; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride>
@@ -962,7 +962,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%stride0> = add ir<%stride>, ir<1>
; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride0>
@@ -1041,7 +1041,7 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx>
@@ -1119,7 +1119,7 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx>
@@ -1199,7 +1199,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p>, ir<%idx>
@@ -1276,7 +1276,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld.base> = getelementptr ir<%p>, ir<%offset>
@@ -1348,7 +1348,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%idx> = add ir<%mul>, ir<%offset>
@@ -1419,7 +1419,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%iv.sdiv32> = sdiv ir<%iv>, ir<32>
; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<%stride>
@@ -1491,7 +1491,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%iv.sdiv32> = sdiv ir<%iv>, ir<32>
; CHECK-NEXT: EMIT ir<%gep.uni> = getelementptr ir<%p.uni>, ir<%iv.sdiv32>
@@ -1571,7 +1571,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -1641,7 +1641,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -1712,7 +1712,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -1785,7 +1785,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -1856,7 +1856,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -1960,7 +1960,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -2037,7 +2037,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%base> = getelementptr ir<%p>, ir<%idx>
@@ -2109,7 +2109,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<42>, ir<%idx>
@@ -2179,7 +2179,7 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<%idx>, ir<42>
@@ -2247,7 +2247,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<%idx>, ir<%idx>
@@ -2315,7 +2315,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT-SCALAR ir<%stride> = sext ir<%stride.i32> to i64
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
@@ -2388,7 +2388,7 @@ define void @i1_zext_stride_with_sext_use(ptr noalias %p.out, ptr %p, i1 %stride
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.zext>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -2463,7 +2463,7 @@ define void @i1_sext_stride_with_zext_use(ptr noalias %p.out, ptr %p, i1 %stride
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.sext>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -2536,7 +2536,7 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT-SCALAR ir<%stride> = trunc ir<%stride.i64> to i32
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
@@ -2606,7 +2606,7 @@ define void @trunc_stride_extra_narrow_use(ptr noalias %p.out, ptr %p, i64 %stri
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT-SCALAR ir<%stride> = trunc ir<%stride.i64> to i32
; CHECK-NEXT: EMIT ir<%stride.byte> = mul ir<%stride>, ir<4>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
@@ -2686,7 +2686,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT-SCALAR ir<%iv.trunc> = trunc ir<%iv> to i16
; CHECK-NEXT: EMIT-SCALAR ir<%iv.ext> = sext ir<%iv> to i64
@@ -2773,7 +2773,7 @@ define void @trunc_ext_chained_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.sext>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -2844,7 +2844,7 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%c> = icmp sge ir<%iv>, ir<%x>
; CHECK-NEXT: Successor(s): if
@@ -2924,7 +2924,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<poison>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -2992,7 +2992,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%iv>
@@ -3061,7 +3061,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -3135,7 +3135,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
@@ -3206,7 +3206,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%iv.times.stride> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%idx> = add ir<%iv.times.stride>, ir<42>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-iv-transforms.ll
index a47b56d9aa85a8..7ac44a53f78254 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-iv-transforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-iv-transforms.ll
@@ -306,7 +306,7 @@ define i64 @early_exit_countdown_iv_via_latch_phi(i64 %n, i64 %mask) {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<%n>, ir<-1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<%n>, ir<-1>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN ir<%iv.next> = add nsw ir<%iv>, ir<-1>
; CHECK-NEXT: WIDEN ir<%hi> = shl nsw ir<-1>, ir<%iv.next>
; CHECK-NEXT: WIDEN ir<%or> = or ir<%hi>, ir<%mask>
@@ -389,8 +389,8 @@ define i32 @early_exit_two_ivs_via_latch_phis(i32 %n, i32 %w, ptr %count.out) {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%cnt> = WIDEN-INDUCTION nsw ir<%n>, ir<-1>, vp<[[VP0]]>
-; CHECK-NEXT: ir<%x> = WIDEN-INDUCTION nsw ir<0>, ir<4>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%cnt> = WIDEN-INDUCTION snsw ir<%n>, ir<-1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%x> = WIDEN-INDUCTION snsw ir<0>, ir<4>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN ir<%x.next> = add nsw ir<%x>, ir<4>
; CHECK-NEXT: WIDEN ir<%c> = icmp sge ir<%x.next>, ir<%w>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%c>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-narrow-iv-users.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-narrow-iv-users.ll
index 7cb871ab734e07..892fadbc7fa735 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-narrow-iv-users.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-narrow-iv-users.ll
@@ -19,7 +19,7 @@ define void @narrow_iv_user_chain(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP3]]> * ir<1>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%and> = and vp<[[VP5]]>, ir<-4>
@@ -74,7 +74,7 @@ define void @narrow_iv_user_chain_multiple_levels(ptr noalias %A, ptr noalias %B
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP3]]> * ir<1>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%and> = and vp<[[VP5]]>, ir<-4>
@@ -130,7 +130,7 @@ define void @no_narrowing_if_all_lanes_used(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP3]]> * ir<1>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep.ld> = getelementptr inbounds ir<%A>, vp<[[VP5]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
index 9e70d16a8c005b..c0a7304c0b49a4 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
@@ -14,7 +14,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; INITIAL-NEXT: Successor(s): scalar.ph, vector.ph
; INITIAL-EMPTY:
; INITIAL-NEXT: vector.ph:
-; INITIAL-NEXT: EMIT vp<%incoming.alias.mask> = incoming-alias-mask
+; INITIAL-NEXT: EMIT vp<%incoming.alias.mask> = incoming-alias-mask
; INITIAL-NEXT: Successor(s): vector loop
; INITIAL-EMPTY:
; INITIAL-NEXT: <x1> vector loop: {
@@ -22,7 +22,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; INITIAL-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK
; INITIAL-EMPTY:
; INITIAL-NEXT: vector.body:
-; INITIAL-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; INITIAL-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; INITIAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = and vp<[[VP4]]>, vp<%incoming.alias.mask>
; INITIAL-NEXT: Successor(s): vector.body.split
; INITIAL-EMPTY:
@@ -60,42 +60,42 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: Successor(s): vector.clamped.vf.check
; FINAL-EMPTY:
; FINAL-NEXT: vector.clamped.vf.check:
-; FINAL-NEXT: WIDEN-INTRINSIC vp<[[VP4:%[0-9]+]]> = call llvm.loop.dependence.war.mask(ir<%b1>, ir<%c2>, ir<1>)
-; FINAL-NEXT: EMIT vp<%num.active.lanes> = num-active-lanes vp<[[VP4]]>
+; FINAL-NEXT: WIDEN-INTRINSIC vp<[[VP2:%[0-9]+]]> = call llvm.loop.dependence.war.mask(ir<%b1>, ir<%c2>, ir<1>)
+; FINAL-NEXT: EMIT vp<%num.active.lanes> = num-active-lanes vp<[[VP2]]>
; FINAL-NEXT: EMIT vp<%vf.is.scalar> = icmp ule vp<%num.active.lanes>, ir<1>
-; FINAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = sub ir<-1>, ir<%n>
-; FINAL-NEXT: EMIT vp<%vf.step.overflow> = icmp ult vp<[[VP5]]>, vp<%num.active.lanes>
-; FINAL-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = or vp<%vf.is.scalar>, vp<%vf.step.overflow>
-; FINAL-NEXT: EMIT branch-on-cond vp<[[VP6]]>
+; FINAL-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = sub ir<-1>, ir<%n>
+; FINAL-NEXT: EMIT vp<%vf.step.overflow> = icmp ult vp<[[VP3]]>, vp<%num.active.lanes>
+; FINAL-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = or vp<%vf.is.scalar>, vp<%vf.step.overflow>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; FINAL-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
; FINAL-EMPTY:
; FINAL-NEXT: vector.ph:
; FINAL-NEXT: EMIT vp<%trip.count.minus.1> = sub ir<%n>, ir<1>
-; FINAL-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = sub vp<%num.active.lanes>, ir<1>
-; FINAL-NEXT: EMIT vp<%n.rnd.up> = add ir<%n>, vp<[[VP8]]>
+; FINAL-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = sub vp<%num.active.lanes>, ir<1>
+; FINAL-NEXT: EMIT vp<%n.rnd.up> = add ir<%n>, vp<[[VP6]]>
; FINAL-NEXT: EMIT vp<%n.mod.vf> = urem vp<%n.rnd.up>, vp<%num.active.lanes>
; FINAL-NEXT: EMIT vp<%n.vec> = sub vp<%n.rnd.up>, vp<%n.mod.vf>
-; FINAL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = broadcast vp<%trip.count.minus.1>
-; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = step-vector i64
-; FINAL-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = broadcast vp<%num.active.lanes>
+; FINAL-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = broadcast vp<%trip.count.minus.1>
+; FINAL-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = step-vector i64
+; FINAL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = broadcast vp<%num.active.lanes>
; FINAL-NEXT: Successor(s): vector.body
; FINAL-EMPTY:
; FINAL-NEXT: vector.body:
; FINAL-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; FINAL-NEXT: WIDEN-PHI vp<[[VP12:%[0-9]+]]> = phi [ vp<[[VP10]]>, vector.ph ], [ vp<%vec.ind.next>, vector.body ]
-; FINAL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = icmp ule vp<[[VP12]]>, vp<[[VP9]]>
-; FINAL-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = and vp<[[VP13]]>, vp<[[VP4]]>
+; FINAL-NEXT: WIDEN-PHI vp<[[VP10:%[0-9]+]]> = phi [ vp<[[VP8]]>, vector.ph ], [ vp<%vec.ind.next>, vector.body ]
+; FINAL-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp ule vp<[[VP10]]>, vp<[[VP7]]>
+; FINAL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = and vp<[[VP11]]>, vp<[[VP2]]>
; FINAL-NEXT: CLONE ir<%ptr.a> = getelementptr inbounds ir<%a>, vp<%index>
-; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP14]]>
+; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP12]]>
; FINAL-NEXT: CLONE ir<%ptr.b> = getelementptr inbounds ir<%b>, vp<%index>
-; FINAL-NEXT: WIDEN ir<%ld.b> = load ir<%ptr.b>, vp<[[VP14]]>
+; FINAL-NEXT: WIDEN ir<%ld.b> = load ir<%ptr.b>, vp<[[VP12]]>
; FINAL-NEXT: WIDEN ir<%add> = add ir<%ld.b>, ir<%ld.a>
; FINAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, vp<%index>
-; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP14]]>
+; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP12]]>
; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<%num.active.lanes>
-; FINAL-NEXT: EMIT vp<%vec.ind.next> = add nuw vp<[[VP12]]>, vp<[[VP11]]>
-; FINAL-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
-; FINAL-NEXT: EMIT branch-on-cond vp<[[VP15]]>
+; FINAL-NEXT: EMIT vp<%vec.ind.next> = add nuw vp<[[VP10]]>, vp<[[VP9]]>
+; FINAL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP13]]>
; FINAL-NEXT: Successor(s): middle.block, vector.body
; FINAL-EMPTY:
; FINAL-NEXT: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll
index 673c8f9e561425..3361ee2ee6ae3b 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll
@@ -26,9 +26,9 @@ define void @cast_flags_mixed(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>, vp<[[VP5]]>, vp<%step.add>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<0>, ir<1>, vp<[[VP0]]>, vp<[[VP5]]>, vp<%step.add>
; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = WIDEN-INDUCTION nuw ir<0>, ir<1>, vp<[[VP0]]>, vp<[[VP6]]>, vp<%step.add>.1
-; CHECK-NEXT: EMIT vp<%step.add> = add nsw ir<%iv>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<%step.add> = add snsw ir<%iv>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%step.add>.1 = add nuw vp<[[VP9]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp ule vp<[[VP9]]>, vp<[[VP3]]>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp ule vp<%step.add>.1, vp<[[VP3]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-intrinsic.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-intrinsic.ll
index f60b6b10604872..103566303a26cb 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-intrinsic.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-intrinsic.ll
@@ -9,7 +9,7 @@ define void @scalar_intrinsic_print(i64 %x, ptr %p) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = call i64 @llvm.vscale(){{$}}
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = call i64 @llvm.vscale()
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = shl nuw vp<[[VP2]]>, ir<2>
; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<128>, vp<[[VP3]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
@@ -27,7 +27,7 @@ define void @scalar_intrinsic_print(i64 %x, ptr %p) {
; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
; CHECK-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ vp<[[VP6]]>, vector.ph ], [ vp<%vec.ind.next>, vector.body ]
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, vp<[[VP3]]>
-; CHECK-NEXT: EMIT vp<%vec.ind.next> = add nuw nsw vp<[[VP8]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%vec.ind.next> = add nuw snsw vp<[[VP8]]>, vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP9]]>
; CHECK-NEXT: Successor(s): middle.block, vector.body
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll
index 3b3fd961f69b3d..63a6d667c510e7 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll
@@ -209,7 +209,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-iv) ir<-9223372036854775808>, vp<[[VP11:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
index 4159dccd2d3012..7f6971572ff38e 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
@@ -255,7 +255,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-iv) ir<-9223372036854775808>, ir<%cond>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
index dba35c2143d170..6c50047a6ba104 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
@@ -182,14 +182,14 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%i> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%i> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%i>, ir<5>
; CHECK-NEXT: Successor(s): pred.udiv
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.udiv: {
; CHECK-NEXT: pred.udiv.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%cmp>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%cmp> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.udiv.if, pred.udiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.udiv.if:
@@ -391,14 +391,14 @@ define void @recipe_debug_loc_location(ptr nocapture %src) !dbg !5 {
; CHECK-NEXT: WIDEN ir<%psd> = add nuw nsw ir<%lsd>, ir<23>, !dbg /tmp/s.c:7:3
; CHECK-NEXT: WIDEN ir<%cmp1> = icmp slt ir<%lsd>, ir<100>, !dbg /tmp/s.c:8:3
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%cmp1>, !dbg /tmp/s.c:9:3
-; CHECK-NEXT: WIDEN ir<%cmp2> = icmp sge ir<%lsd>, ir<200>, !dbg /tmp/s.c:10:3
+; CHECK-NEXT: WIDEN ir<%cmp2> = icmp sge ir<%lsd>, ir<200>, !dbg /tmp/s.c:10:3 (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%cmp2>, !dbg /tmp/s.c:11:3
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP7]]>, ir<%cmp1>
; CHECK-NEXT: Successor(s): pred.sdiv
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.sdiv: {
; CHECK-NEXT: pred.sdiv.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]> (!vplan.execution.frequency 13835058055282163711 (75%, estimated))
; CHECK-NEXT: Successor(s): pred.sdiv.if, pred.sdiv.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.sdiv.if:
@@ -434,10 +434,10 @@ define void @recipe_debug_loc_location(ptr nocapture %src) !dbg !5 {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %if.end ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; CHECK-NEXT: IR %isd = getelementptr inbounds i32, ptr %src, i64 %iv, !dbg !{{[0-9]+}}
-; CHECK-NEXT: IR %lsd = load i32, ptr %isd, align 4, !dbg !{{[0-9]+}}
-; CHECK-NEXT: IR %psd = add nuw nsw i32 %lsd, 23, !dbg !{{[0-9]+}}
-; CHECK-NEXT: IR %cmp1 = icmp slt i32 %lsd, 100, !dbg !{{[0-9]+}}
+; CHECK-NEXT: IR %isd = getelementptr inbounds i32, ptr %src, i64 %iv, !dbg !24
+; CHECK-NEXT: IR %lsd = load i32, ptr %isd, align 4, !dbg !25
+; CHECK-NEXT: IR %psd = add nuw nsw i32 %lsd, 23, !dbg !26
+; CHECK-NEXT: IR %cmp1 = icmp slt i32 %lsd, 100, !dbg !27
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
@@ -803,7 +803,7 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.call: {
; CHECK-NEXT: pred.call.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%ifcond>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%ifcond> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
; CHECK-NEXT: Successor(s): pred.call.if, pred.call.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.call.if:
@@ -819,7 +819,7 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
; CHECK-NEXT: Successor(s): if.then.1
; CHECK-EMPTY:
; CHECK-NEXT: if.then.1:
-; CHECK-NEXT: WIDEN ir<%fadd> = fadd vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK-NEXT: WIDEN ir<%fadd> = fadd vp<[[VP6]]>, vp<[[VP7]]> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
; CHECK-NEXT: BLEND ir<%st.value> = ir<%ld.value> ir<%fadd>/ir<%ifcond>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr inbounds ir<%dest>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds float, ir<%st.addr>, ir<1>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-scev-address-idioms.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-scev-address-idioms.ll
index e0ac1a6e6921dc..7866a425ff13f0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-scev-address-idioms.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-scev-address-idioms.ll
@@ -20,7 +20,7 @@ define void @bitcast_noop(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP4]]>
@@ -83,13 +83,13 @@ define void @ashr_zero(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%iv>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep>, ir<1>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP4]]>
; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%B>, ir<%iv>
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.b>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%l>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.b>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%l>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<100>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
@@ -99,7 +99,7 @@ define void @ashr_zero(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<101>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -146,7 +146,7 @@ define void @ashr_of_add_of_shl(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%s> = shl nsw ir<%iv>, ir<3>
; CHECK-NEXT: EMIT ir<%a> = add ir<%s>, ir<8>
; CHECK-NEXT: EMIT ir<%idx> = ashr ir<%a>, ir<3>
@@ -213,7 +213,7 @@ define void @ashr_variable(ptr noalias %A, ptr noalias %B, i64 %n) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%idx> = ashr ir<%iv>, ir<%n>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep>
@@ -277,7 +277,7 @@ define void @ashr_bitwidth(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%s> = shl nsw ir<%iv>, ir<64>
; CHECK-NEXT: EMIT ir<%idx> = ashr ir<%s>, ir<64>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
@@ -342,7 +342,7 @@ define void @usub_sat(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%idx> = call i64 @llvm.usub.sat.i64(ir<%iv>, ir<0>)
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep>
@@ -405,7 +405,7 @@ define void @uadd_sat(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%idx> = call i64 @llvm.uadd.sat.i64(ir<%iv>, ir<0>)
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep>
@@ -469,7 +469,7 @@ define void @usub_sat_clamped(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%idx> = call i64 @llvm.usub.sat.i64(ir<4>, ir<%iv>)
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep>
@@ -532,7 +532,7 @@ define void @ptrtoaddr_gep(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%p> = getelementptr inbounds ir<null>, ir<%iv>
; CHECK-NEXT: EMIT-SCALAR ir<%idx> = ptrtoaddr ir<%p> to i64
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
@@ -599,7 +599,7 @@ define void @ptrtoint_gep(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%p> = getelementptr inbounds ir<null>, ir<%iv>
; CHECK-NEXT: EMIT-SCALAR ir<%idx> = ptrtoint ir<%p> to i64
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
@@ -665,7 +665,7 @@ define void @xor_signmask(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%x> = xor ir<%iv>, ir<-9223372036854775808>
; CHECK-NEXT: EMIT ir<%idx> = add ir<%x>, ir<-9223372036854775808>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
@@ -730,7 +730,7 @@ define void @xor_not_signmask(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%idx> = xor ir<%iv>, ir<12>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep>
@@ -793,7 +793,7 @@ define void @select_constant_cond(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%idx> = select ir<true>, ir<%iv>, ir<0>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep>
@@ -856,7 +856,7 @@ define void @select_equal_arms(ptr noalias %A, ptr noalias %B, i64 %n) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%a> = add ir<%iv>, ir<4>
; CHECK-NEXT: EMIT ir<%b> = or disjoint ir<%iv>, ir<4>
; CHECK-NEXT: EMIT ir<%c> = icmp ult ir<%iv>, ir<%n>
@@ -925,7 +925,7 @@ define void @select_different_arms(ptr noalias %A, ptr noalias %B, i64 %n) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%a> = add ir<%iv>, ir<4>
; CHECK-NEXT: EMIT ir<%b> = add ir<%iv>, ir<8>
; CHECK-NEXT: EMIT ir<%c> = icmp ult ir<%iv>, ir<%n>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
index b8746ef08275ed..5c3b5a0d5e484d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
@@ -37,7 +37,7 @@ define void @sink1(i32 %k, i32 %x) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP10]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP10]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
@@ -52,13 +52,13 @@ define void @sink1(i32 %k, i32 %x) {
; CHECK-NEXT: Successor(s): if.0
; CHECK-EMPTY:
; CHECK-NEXT: if.0:
-; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[VP11]]>, ir<10>
+; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[VP11]]>, ir<10> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = shl ir<%add>, ir<1>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP10]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP10]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -366,7 +366,7 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<21>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<21>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = DERIVED-IV ir<21> + vp<[[VP4]]> * ir<1>
@@ -377,7 +377,7 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -469,7 +469,7 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
@@ -588,7 +588,7 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
@@ -610,7 +610,7 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP12]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP12]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -715,7 +715,7 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
@@ -736,7 +736,7 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -863,12 +863,12 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: loop.3:
; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%c.0>
-; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[VP10]]>, vp<[[VP11]]>
+; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[VP10]]>, vp<[[VP11]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP12]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP12]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -1163,14 +1163,14 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%c>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%c> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>
; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1>
-; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0>
-; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2>
+; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
+; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
@@ -1254,7 +1254,7 @@ define void @sinking_requires_duplication(ptr %addr) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%pred>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%pred> (!vplan.execution.frequency 6701356245527298047 (36.33%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -1336,14 +1336,14 @@ define void @merge_with_dead_gep_between_regions(i32 %n, i32 %k, ptr noalias %sr
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<%n>, ir<-1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION snsw ir<%n>, ir<-1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%n> + vp<[[VP4]]> * ir<-1>
; CHECK-NEXT: WIDEN ir<%cond> = icmp ult ir<%iv>, ir<%k>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%cond>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%cond> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -1442,7 +1442,7 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%c.1>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%c.1> (!vplan.execution.frequency 11529215046068469759 (62.5%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/widen_mem_idioms.ll b/llvm/test/Transforms/LoopVectorize/VPlan/widen_mem_idioms.ll
index 6c9bc6339c9973..8ee1fae507de28 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/widen_mem_idioms.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/widen_mem_idioms.ll
@@ -20,7 +20,7 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%gep.indices> = getelementptr inbounds ir<%indices>, ir<%iv>
; CHECK-NEXT: EMIT-SCALAR ir<%l.idx> = load ir<%gep.indices>
; CHECK-NEXT: EMIT-SCALAR ir<%idxprom1> = zext ir<%l.idx> to i64
@@ -102,7 +102,7 @@ define void @reduc_store_inside_unrolled(ptr noalias %dst, ptr noalias readonly
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<2>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<2>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum> = phi (add) ir<0>, ir<%sum.2>
; CHECK-NEXT: EMIT ir<%gep.src> = getelementptr inbounds ir<%src>, ir<%iv>
; CHECK-NEXT: EMIT-SCALAR ir<%0> = load ir<%gep.src>
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
index 729337631b002e..27f04c47a4c3fe 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
@@ -20,7 +20,7 @@ define void @wide_or_replaced_with_add_vpinstruction(ptr %src, ptr noalias %dst)
; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %iv.next = add nuw nsw i64 %iv, 1
; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %exitcond = icmp eq i64 %iv.next, 32
; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction: br i1 %exitcond, label %exit, label %loop.header
-; CHECK: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 2: CLONE ir<%g.src> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i64, ir<%g.src>, ir<1>
@@ -30,7 +30,7 @@ define void @wide_or_replaced_with_add_vpinstruction(ptr %src, ptr noalias %dst)
; CHECK: Cost of 1 for VF 2: EMIT ir<%or> = add ir<%iv.4>, ir<1>
; CHECK: Cost of 0 for VF 2: CLONE ir<%g.dst> = getelementptr ir<%dst>, ir<%or>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer i64, ir<%g.dst>, ir<1>
-; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP6]]>, ir<%iv.4>, ir<%c>
+; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP6]]>, ir<%iv.4>, ir<%c> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK: Cost of 1 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: vector loop backedge
@@ -43,7 +43,7 @@ define void @wide_or_replaced_with_add_vpinstruction(ptr %src, ptr noalias %dst)
; CHECK: Cost of 0 for VF 2: IR %c = icmp ule i64 %l, 128
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<32>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost of 1 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK: Cost of 1 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: CLONE ir<%g.src> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds i64, ir<%g.src>, ir<1>
@@ -53,7 +53,7 @@ define void @wide_or_replaced_with_add_vpinstruction(ptr %src, ptr noalias %dst)
; CHECK: Cost of 1 for VF 4: EMIT ir<%or> = add ir<%iv.4>, ir<1>
; CHECK: Cost of 0 for VF 4: CLONE ir<%g.dst> = getelementptr ir<%dst>, ir<%or>
; CHECK: Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer i64, ir<%g.dst>, ir<1>
-; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP6]]>, ir<%iv.4>, ir<%c>
+; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP6]]>, ir<%iv.4>, ir<%c> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK: Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK: Cost of 1 for VF 4: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: vector loop backedge
@@ -200,10 +200,10 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = or vp<[[VP9]]>, vp<[[VP10]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = or vp<[[VP12]]>, vp<[[VP11]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = not vp<[[VP13]]>
-; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP8]]>, ir<1>, vp<[[VP11]]>
-; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP8]]>, ir<0>, vp<[[VP10]]>
-; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP8]]>, ir<42>, vp<[[VP9]]>
-; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP8]]>, ir<2>, vp<[[VP14]]>
+; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP8]]>, ir<1>, vp<[[VP11]]> (!vplan.execution.frequency 4611686018427387903 (25%, estimated))
+; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP8]]>, ir<0>, vp<[[VP10]]> (!vplan.execution.frequency 4611686020574871552 (25%, estimated))
+; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP8]]>, ir<42>, vp<[[VP9]]> (!vplan.execution.frequency 4611686017353646080 (25%, estimated))
+; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP8]]>, ir<2>, vp<[[VP14]]> (!vplan.execution.frequency 4611686017353646080 (25%, estimated))
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1:%[0-9]+]]>
; CHECK: Cost of 1 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: vector loop backedge
@@ -226,10 +226,10 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP12]]> = or vp<[[VP9]]>, vp<[[VP10]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP13]]> = or vp<[[VP12]]>, vp<[[VP11]]>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP14]]> = not vp<[[VP13]]>
-; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP8]]>, ir<1>, vp<[[VP11]]>
-; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP8]]>, ir<0>, vp<[[VP10]]>
-; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP8]]>, ir<42>, vp<[[VP9]]>
-; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP8]]>, ir<2>, vp<[[VP14]]>
+; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP8]]>, ir<1>, vp<[[VP11]]> (!vplan.execution.frequency 4611686018427387903 (25%, estimated))
+; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP8]]>, ir<0>, vp<[[VP10]]> (!vplan.execution.frequency 4611686020574871552 (25%, estimated))
+; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP8]]>, ir<42>, vp<[[VP9]]> (!vplan.execution.frequency 4611686017353646080 (25%, estimated))
+; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP8]]>, ir<2>, vp<[[VP14]]> (!vplan.execution.frequency 4611686017353646080 (25%, estimated))
; CHECK: Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK: Cost of 1 for VF 4: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: vector loop backedge
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
index cf081d726faaac..dcd09da6387ad3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
@@ -217,7 +217,7 @@ define i64 @test_pr62954_scalar_epilogue_required(ptr %A, ptr noalias %B, ptr %C
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 4)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 36
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
index 6f5b9d4403af12..b5e850624354d6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
@@ -512,7 +512,7 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
@@ -555,7 +555,7 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP6]], <8 x i1> [[TMP2]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; ENABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
@@ -732,7 +732,7 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
@@ -850,7 +850,7 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
; ENABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; ENABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
@@ -1133,7 +1133,7 @@ define void @unconditional_strided1_optsize_unknown_tc(ptr noalias nocapture rea
; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP49]], ptr align 1 [[TMP50]], <8 x i1> [[TMP0]])
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
@@ -2610,7 +2610,7 @@ define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr n
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE62]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE62]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP159]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
@@ -2657,7 +2657,7 @@ define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr n
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP9]], <16 x i8> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP6]], <16 x i1> [[INTERLEAVED_MASK]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; ENABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
@@ -3039,7 +3039,7 @@ define void @unconditional_masked_strided2_unknown_tc(ptr noalias nocapture read
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE60]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE60]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP157]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
index ce1b579dba671a..7467cd6a3b18dd 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
@@ -157,7 +157,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP4]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP4]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP6]], ptr [[TMP5]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE]]
@@ -166,7 +166,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF1]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP8]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP8]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP10]], ptr [[TMP9]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE2]]
@@ -175,7 +175,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF3]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP12]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP12]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP14]], ptr [[TMP13]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE4]]
@@ -184,7 +184,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF5]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP16]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP16]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP18]], ptr [[TMP17]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE6]]
@@ -195,7 +195,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF8]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP20]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP21]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP21]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP23]], ptr [[TMP22]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE9]]
@@ -203,7 +203,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF10]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP20]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP24]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP24]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP26]], ptr [[TMP25]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE11]]
@@ -211,7 +211,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF12]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP20]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP27]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP27]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP29]], ptr [[TMP28]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE13]]
@@ -219,13 +219,13 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF14]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[TMP20]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP30]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP30]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP32]], ptr [[TMP31]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE15]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE15]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
index c6006019afc8fe..95b628dbaa8142 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
@@ -539,7 +539,7 @@ define i64 @find_iv_reduction(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[CLAMPED_HEADER_MASK]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP12]] = select <4 x i1> [[TMP17]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-iv-interleave.ll b/llvm/test/Transforms/LoopVectorize/find-last-iv-interleave.ll
index 1e7635ecaed30f..3737f438d8dcff 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last-iv-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last-iv-interleave.ll
@@ -90,7 +90,7 @@ define i64 @findlast_iv_start5_interleave(ptr %a, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 5, i64 6, i64 7, i64 8>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add nuw i64 5, [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i64 4
@@ -167,7 +167,7 @@ define i64 @findlast_iv_step2_interleave(ptr %a, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 2, i64 4, i64 6>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP38:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP39:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 8)
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 8)
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl nuw i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 2
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
index 6becee93c41176..9d9d732be3db92 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
@@ -28,7 +28,7 @@ define i64 @findlast_shl_expr(ptr %a, i64 %n) {
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -83,7 +83,7 @@ define i64 @findlast_mul_expr(ptr %a, i64 %n) {
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -138,7 +138,7 @@ define i64 @findlast_or_expr(ptr %a, i64 %n) {
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -379,7 +379,7 @@ define i32 @findlast_wider_iv_with_vector_users(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: [[TMP12]] = select i1 [[TMP11]], <4 x i1> [[TMP10]], <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP13]] = select i1 [[TMP11]], <4 x i32> [[TMP8]], <4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], splat (i32 4)
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -603,7 +603,7 @@ define i64 @findlast_sub_n_expr(ptr %a, i64 %n) {
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -660,7 +660,7 @@ define i64 @findlast_sub_iv_as_lhs(ptr %a, i64 %n) {
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
index 5b8e09b18fcd42..5f6f8d25424c67 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
@@ -1134,8 +1134,8 @@ define i32 @findlast_iv_step_nonzero_unknown_sign(ptr %a, i32 %x) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[STEP]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = mul nsw <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP1:%.*]] = shl nsw i32 [[STEP]], 2
+; CHECK-NEXT: [[TMP0:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = shl i32 [[STEP]], 2
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1152,7 +1152,7 @@ define i32 @findlast_iv_step_nonzero_unknown_sign(ptr %a, i32 %x) {
; CHECK-NEXT: [[TMP7]] = select i1 [[TMP6]], <4 x i1> [[TMP5]], <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP8]] = select i1 [[TMP6]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index bb9a5e7cd9367f..88052eb7543d2b 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -962,7 +962,7 @@ define i32 @PR27246() {
; UNROLL-NO-IC: [[VECTOR_BODY]]:
; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 -4)
+; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)
; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[STEP_ADD]], splat (i32 -4)
; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/force-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/force-instruction-cost.ll
index 87731eab81a3e4..0182e7b2a38978 100644
--- a/llvm/test/Transforms/LoopVectorize/force-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/force-instruction-cost.ll
@@ -5,7 +5,7 @@
define void @int_induction(ptr noalias %dst, i64 %n) {
; FORCED1-LABEL: 'int_induction'
-; FORCED1: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; FORCED1: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; FORCED1: Cost of 1 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; FORCED1: Cost of 1 for VF 2: CLONE ir<%gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; FORCED1: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i64, ir<%gep>, ir<1>
@@ -14,7 +14,7 @@ define void @int_induction(ptr noalias %dst, i64 %n) {
; FORCED1: Cost of 0 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
;
; FORCED3-LABEL: 'int_induction'
-; FORCED3: Cost of 3 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; FORCED3: Cost of 3 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; FORCED3: Cost of 3 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; FORCED3: Cost of 3 for VF 2: CLONE ir<%gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; FORCED3: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i64, ir<%gep>, ir<1>
@@ -39,7 +39,7 @@ exit:
define void @fp_induction_scalar_users(ptr noalias %dst, i64 %n) {
; FORCED1-LABEL: 'fp_induction_scalar_users'
-; FORCED1: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; FORCED1: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; FORCED1: Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; FORCED1: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; FORCED1: Cost of 1 for VF 2: WIDEN ir<%idx> = mul ir<%iv>, ir<3>
@@ -49,7 +49,7 @@ define void @fp_induction_scalar_users(ptr noalias %dst, i64 %n) {
; FORCED1: Cost of 0 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
;
; FORCED3-LABEL: 'fp_induction_scalar_users'
-; FORCED3: Cost of 3 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; FORCED3: Cost of 3 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; FORCED3: Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; FORCED3: Cost of 3 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; FORCED3: Cost of 3 for VF 2: WIDEN ir<%idx> = mul ir<%iv>, ir<3>
diff --git a/llvm/test/Transforms/LoopVectorize/induction-cost.ll b/llvm/test/Transforms/LoopVectorize/induction-cost.ll
index f028364da45a1f..972f4ceb137fd0 100644
--- a/llvm/test/Transforms/LoopVectorize/induction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction-cost.ll
@@ -5,13 +5,13 @@
define void @int_induction(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'int_induction'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
; VF2: Cost for VF 2: 6 (Estimated cost per lane: 3)
;
; VF4-LABEL: 'int_induction'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
; VF4: Cost for VF 4: 6 (Estimated cost per lane: 1.5)
@@ -94,14 +94,14 @@ exit:
define void @trunc_and_wide_induction(ptr noalias %dst, ptr noalias %dst2, i64 %n) {
; VF2-LABEL: 'trunc_and_wide_induction'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 2 for VF 2: ir<%iv>.1 = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> (truncated to i32)
; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
; VF2: Cost for VF 2: 9 (Estimated cost per lane: 4.5)
;
; VF4-LABEL: 'trunc_and_wide_induction'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 2 for VF 4: ir<%iv>.1 = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> (truncated to i32)
; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
@@ -129,7 +129,7 @@ exit:
; and the scalar steps materialize one FAdd per lane, except for the first.
define void @fp_induction_scalar_users(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_scalar_users'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
@@ -137,7 +137,7 @@ define void @fp_induction_scalar_users(ptr noalias %dst, i64 %n) {
; VF2: Cost for VF 2: 10 (Estimated cost per lane: 5)
;
; VF4-LABEL: 'fp_induction_scalar_users'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
@@ -166,7 +166,7 @@ exit:
; FSub to compute the per-lane values.
define void @fp_induction_fsub_scalar_users(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_fsub_scalar_users'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<1.000000e+02> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
@@ -174,7 +174,7 @@ define void @fp_induction_fsub_scalar_users(ptr noalias %dst, i64 %n) {
; VF2: Cost for VF 2: 11 (Estimated cost per lane: 5.5)
;
; VF4-LABEL: 'fp_induction_fsub_scalar_users'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<1.000000e+02> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
@@ -202,7 +202,7 @@ exit:
; Same as @fp_induction_scalar_users, but with a loop-invariant step.
define void @fp_induction_invariant_step_scalar_users(ptr noalias %dst, float %step, i64 %n) {
; VF2-LABEL: 'fp_induction_invariant_step_scalar_users'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
@@ -210,7 +210,7 @@ define void @fp_induction_invariant_step_scalar_users(ptr noalias %dst, float %s
; VF2: Cost for VF 2: 12 (Estimated cost per lane: 6)
;
; VF4-LABEL: 'fp_induction_invariant_step_scalar_users'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step>
; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
@@ -242,7 +242,7 @@ exit:
; regardless of fast-math flags.
define void @fp_induction_no_fastmath_unit_step_zero_start(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_no_fastmath_unit_step_zero_start'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
@@ -250,7 +250,7 @@ define void @fp_induction_no_fastmath_unit_step_zero_start(ptr noalias %dst, i64
; VF2: Cost for VF 2: 10 (Estimated cost per lane: 5)
;
; VF4-LABEL: 'fp_induction_no_fastmath_unit_step_zero_start'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
@@ -282,7 +282,7 @@ exit:
; it may be -0.0, and "fadd 0.0, X" doesn't fold to X in that case.
define void @fp_induction_no_fastmath_zero_start_invariant_step(ptr noalias %dst, float %step, i64 %n) {
; VF2-LABEL: 'fp_induction_no_fastmath_zero_start_invariant_step'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
@@ -290,7 +290,7 @@ define void @fp_induction_no_fastmath_zero_start_invariant_step(ptr noalias %dst
; VF2: Cost for VF 2: 12 (Estimated cost per lane: 6)
;
; VF4-LABEL: 'fp_induction_no_fastmath_zero_start_invariant_step'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step>
; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
@@ -320,7 +320,7 @@ exit:
; instruction.
define void @fp_induction_fsub_zero_start(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_fsub_zero_start'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
@@ -328,7 +328,7 @@ define void @fp_induction_fsub_zero_start(ptr noalias %dst, i64 %n) {
; VF2: Cost for VF 2: 11 (Estimated cost per lane: 5.5)
;
; VF4-LABEL: 'fp_induction_fsub_zero_start'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
@@ -356,7 +356,7 @@ exit:
; Start of -0.0 so the FAdd can be folded.
define void @fp_induction_neg_zero_start_neg_step(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_neg_zero_start_neg_step'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<-0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
@@ -364,7 +364,7 @@ define void @fp_induction_neg_zero_start_neg_step(ptr noalias %dst, i64 %n) {
; VF2: Cost for VF 2: 11 (Estimated cost per lane: 5.5)
;
; VF4-LABEL: 'fp_induction_neg_zero_start_neg_step'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<-0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00>
; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
@@ -393,7 +393,7 @@ exit:
; FAdd cannot be folded.
define void @fp_induction_pos_zero_start_neg_step(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_pos_zero_start_neg_step'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
@@ -401,7 +401,7 @@ define void @fp_induction_pos_zero_start_neg_step(ptr noalias %dst, i64 %n) {
; VF2: Cost for VF 2: 12 (Estimated cost per lane: 6)
;
; VF4-LABEL: 'fp_induction_pos_zero_start_neg_step'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00>
; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
@@ -429,7 +429,7 @@ exit:
; Same as @fp_induction_pos_zero_start_neg_step, but with a positive step.
define void @fp_induction_pos_zero_start_pos_step(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_pos_zero_start_pos_step'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<2.000000e+00>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<2.000000e+00>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
@@ -437,7 +437,7 @@ define void @fp_induction_pos_zero_start_pos_step(ptr noalias %dst, i64 %n) {
; VF2: Cost for VF 2: 12 (Estimated cost per lane: 6)
;
; VF4-LABEL: 'fp_induction_pos_zero_start_pos_step'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<2.000000e+00>
; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<2.000000e+00>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
@@ -466,7 +466,7 @@ exit:
; unit step folds the FMul away.
define void @fp_induction_invariant_start(ptr noalias %dst, float %start, i64 %n) {
; VF2-LABEL: 'fp_induction_invariant_start'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
@@ -474,7 +474,7 @@ define void @fp_induction_invariant_start(ptr noalias %dst, float %start, i64 %n
; VF2: Cost for VF 2: 11 (Estimated cost per lane: 5.5)
;
; VF4-LABEL: 'fp_induction_invariant_start'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
@@ -503,13 +503,13 @@ exit:
; loop and nothing is charged for it.
define float @fp_induction_used_outside_loop_only(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_used_outside_loop_only'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
; VF2: Cost for VF 2: 8 (Estimated cost per lane: 4)
;
; VF4-LABEL: 'fp_induction_used_outside_loop_only'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
; VF4: Cost for VF 4: 8 (Estimated cost per lane: 2)
@@ -535,7 +535,7 @@ exit:
; and the scalar steps are generated and need to be accounted for.
define void @fp_induction_wide_and_scalar_users(ptr noalias %dst, ptr noalias %dst2, i64 %n) {
; VF2-LABEL: 'fp_induction_wide_and_scalar_users'
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 2 for VF 2: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00>
@@ -545,7 +545,7 @@ define void @fp_induction_wide_and_scalar_users(ptr noalias %dst, ptr noalias %d
; VF2: Cost for VF 2: 13 (Estimated cost per lane: 6.5)
;
; VF4-LABEL: 'fp_induction_wide_and_scalar_users'
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 2 for VF 4: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4: Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00>
@@ -580,7 +580,7 @@ define void @fp_induction_predicated(ptr noalias %dst, ptr noalias %cond, i64 %n
; VF2-LABEL: 'fp_induction_predicated'
; VF2: Cost of 1 for VF 2: profitable to scalarize store float %fp.iv, ptr %gep, align 4
; VF2: Precomputed costs for VF 2: 3
-; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00>
; VF2: Cost of 0.5 for VF 2: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1.000000e+00>, vp<[[VP0]]>
@@ -591,7 +591,7 @@ define void @fp_induction_predicated(ptr noalias %dst, ptr noalias %cond, i64 %n
; VF4-LABEL: 'fp_induction_predicated'
; VF4: Cost of 2 for VF 4: profitable to scalarize store float %fp.iv, ptr %gep, align 4
; VF4: Precomputed costs for VF 4: 6
-; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw snsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00>
; VF4: Cost of 1.5 for VF 4: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1.000000e+00>, vp<[[VP0]]>
diff --git a/llvm/test/Transforms/LoopVectorize/induction-step.ll b/llvm/test/Transforms/LoopVectorize/induction-step.ll
index 7aeebb469a839e..ad4828438b65a7 100644
--- a/llvm/test/Transforms/LoopVectorize/induction-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction-step.ll
@@ -37,9 +37,9 @@ define void @induction_with_global(i32 %init, ptr noalias nocapture %A, i32 %N)
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[TMP0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP7:%.*]] = mul nsw <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[BROADCAST_SPLAT2]]
-; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <8 x i32> [[BROADCAST_SPLAT]], [[TMP7]]
-; CHECK-NEXT: [[TMP8:%.*]] = shl nsw i32 [[TMP0]], 3
+; CHECK-NEXT: [[TMP11:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <8 x i32> [[BROADCAST_SPLAT]], [[TMP11]]
+; CHECK-NEXT: [[TMP8:%.*]] = shl i32 [[TMP0]], 3
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i32> poison, i32 [[TMP8]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT3]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -49,7 +49,7 @@ define void @induction_with_global(i32 %init, ptr noalias nocapture %A, i32 %N)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: store <8 x i32> [[VEC_IND]], ptr [[TMP9]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -126,9 +126,9 @@ define i32 @induction_with_loop_inv(i32 %init, ptr noalias nocapture %A, i32 %N,
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[J_012]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = mul nsw <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[BROADCAST_SPLAT2]]
-; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <8 x i32> [[BROADCAST_SPLAT]], [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = shl nsw i32 [[J_012]], 3
+; CHECK-NEXT: [[TMP10:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <8 x i32> [[BROADCAST_SPLAT]], [[TMP10]]
+; CHECK-NEXT: [[TMP7:%.*]] = shl i32 [[J_012]], 3
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i32> poison, i32 [[TMP7]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT3]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -138,7 +138,7 @@ define i32 @induction_with_loop_inv(i32 %init, ptr noalias nocapture %A, i32 %N,
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: store <8 x i32> [[VEC_IND]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/induction-wrapflags.ll b/llvm/test/Transforms/LoopVectorize/induction-wrapflags.ll
index 230d3c1968c647..feda399018db05 100644
--- a/llvm/test/Transforms/LoopVectorize/induction-wrapflags.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction-wrapflags.ll
@@ -341,7 +341,7 @@ define i8 @add_induction_nsw_lane_offset_overflow(i8 %start, i8 %n) {
; CHECK-NEXT: [[TMP6:%.*]] = add i8 [[START]], [[TMP5]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[START]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION:%.*]] = add nuw nsw <4 x i8> [[BROADCAST_SPLAT]], <i8 0, i8 60, i8 120, i8 -76>
+; CHECK-NEXT: [[INDUCTION:%.*]] = add nuw <4 x i8> [[BROADCAST_SPLAT]], <i8 0, i8 60, i8 120, i8 -76>
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -349,7 +349,7 @@ define i8 @add_induction_nsw_lane_offset_overflow(i8 %start, i8 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP7]] = add <4 x i8> [[VEC_PHI]], [[VEC_IND]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i8> [[VEC_IND]], splat (i8 -16)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 -16)
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -529,7 +529,7 @@ define void @add_induction_nsw_constant_step_lane_offset_overflow(ptr noalias %d
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[START]], [[TMP3]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[START]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 -1073741830, i32 2147483636, i32 1073741806>
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 -1073741830, i32 2147483636, i32 1073741806>
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -537,7 +537,7 @@ define void @add_induction_nsw_constant_step_lane_offset_overflow(ptr noalias %d
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP5]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 -24)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 -24)
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index 193011f56e2298..88c4b519e34e97 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -110,7 +110,7 @@ define void @multi_int_induction(ptr %A, i32 %N) {
; UNROLL: vector.body:
; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 190, i32 191>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NEXT: [[STEP_ADD:%.*]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
+; UNROLL-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
; UNROLL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 2
; UNROLL-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP3]], align 4
@@ -155,7 +155,7 @@ define void @multi_int_induction(ptr %A, i32 %N) {
; UNROLL-NO-IC: vector.body:
; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 190, i32 191>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
+; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 2
; UNROLL-NO-IC-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP4]], align 4
@@ -200,7 +200,7 @@ define void @multi_int_induction(ptr %A, i32 %N) {
; INTERLEAVE: vector.body:
; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 190, i32 191, i32 192, i32 193>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 4
; INTERLEAVE-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP3]], align 4
@@ -1574,12 +1574,12 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i64 1
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP26]], align 1, !alias.scope [[META17:![0-9]+]]
-; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP15]], align 1, !alias.scope [[META17]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP26]], align 1, !alias.scope [[META17:![0-9]+]]
+; CHECK-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP15]], align 1, !alias.scope [[META17]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P]], i64 [[INDEX]], i32 1
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP10]], i32 1
-; CHECK-NEXT: store i32 [[TMP16]], ptr [[TMP18]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
-; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP19]], align 1, !alias.scope [[META20]], !noalias [[META17]]
+; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP18]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
+; CHECK-NEXT: store i32 [[TMP27]], ptr [[TMP19]], align 1, !alias.scope [[META20]], !noalias [[META17]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1636,12 +1636,12 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
; IND-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
; IND-NEXT: [[TMP13:%.*]] = extractelement <2 x i64> [[TMP10]], i64 1
; IND-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP13]]
-; IND-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP12]], align 1, !alias.scope [[META17:![0-9]+]]
-; IND-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP14]], align 1, !alias.scope [[META17]]
+; IND-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP12]], align 1, !alias.scope [[META17:![0-9]+]]
+; IND-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP14]], align 1, !alias.scope [[META17]]
; IND-NEXT: [[TMP17:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P]], i64 [[INDEX]], i32 1
; IND-NEXT: [[TMP18:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP9]], i32 1
-; IND-NEXT: store i32 [[TMP16]], ptr [[TMP17]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
-; IND-NEXT: store i32 [[TMP25]], ptr [[TMP18]], align 1, !alias.scope [[META20]], !noalias [[META17]]
+; IND-NEXT: store i32 [[TMP25]], ptr [[TMP17]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
+; IND-NEXT: store i32 [[TMP26]], ptr [[TMP18]], align 1, !alias.scope [[META20]], !noalias [[META17]]
; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; IND-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)
; IND-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1706,18 +1706,18 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
; UNROLL-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP20]]
; UNROLL-NEXT: [[TMP35:%.*]] = extractelement <2 x i64> [[TMP13]], i64 1
; UNROLL-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP35]]
-; UNROLL-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP15]], align 1, !alias.scope [[META17:![0-9]+]]
-; UNROLL-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP17]], align 1, !alias.scope [[META17]]
-; UNROLL-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP19]], align 1, !alias.scope [[META17]]
-; UNROLL-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP21]], align 1, !alias.scope [[META17]]
+; UNROLL-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP15]], align 1, !alias.scope [[META17:![0-9]+]]
+; UNROLL-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP17]], align 1, !alias.scope [[META17]]
+; UNROLL-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP19]], align 1, !alias.scope [[META17]]
+; UNROLL-NEXT: [[TMP38:%.*]] = load i32, ptr [[TMP21]], align 1, !alias.scope [[META17]]
; UNROLL-NEXT: [[TMP26:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P]], i64 [[INDEX]], i32 1
; UNROLL-NEXT: [[TMP27:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP9]], i32 1
; UNROLL-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP10]], i32 1
; UNROLL-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP11]], i32 1
-; UNROLL-NEXT: store i32 [[TMP23]], ptr [[TMP26]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
-; UNROLL-NEXT: store i32 [[TMP24]], ptr [[TMP27]], align 1, !alias.scope [[META20]], !noalias [[META17]]
-; UNROLL-NEXT: store i32 [[TMP25]], ptr [[TMP28]], align 1, !alias.scope [[META20]], !noalias [[META17]]
-; UNROLL-NEXT: store i32 [[TMP37]], ptr [[TMP29]], align 1, !alias.scope [[META20]], !noalias [[META17]]
+; UNROLL-NEXT: store i32 [[TMP24]], ptr [[TMP26]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
+; UNROLL-NEXT: store i32 [[TMP25]], ptr [[TMP27]], align 1, !alias.scope [[META20]], !noalias [[META17]]
+; UNROLL-NEXT: store i32 [[TMP37]], ptr [[TMP28]], align 1, !alias.scope [[META20]], !noalias [[META17]]
+; UNROLL-NEXT: store i32 [[TMP38]], ptr [[TMP29]], align 1, !alias.scope [[META20]], !noalias [[META17]]
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; UNROLL-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)
; UNROLL-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1782,18 +1782,18 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP21]]
; UNROLL-NO-IC-NEXT: [[TMP37:%.*]] = extractelement <2 x i64> [[TMP14]], i64 1
; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP37]]
-; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP18]], align 1, !alias.scope [[META17:![0-9]+]]
-; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP20]], align 1, !alias.scope [[META17]]
-; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP22]], align 1, !alias.scope [[META17]]
-; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP38]], align 1, !alias.scope [[META17]]
+; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP18]], align 1, !alias.scope [[META17:![0-9]+]]
+; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP20]], align 1, !alias.scope [[META17]]
+; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP22]], align 1, !alias.scope [[META17]]
+; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = load i32, ptr [[TMP38]], align 1, !alias.scope [[META17]]
; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P]], i64 [[INDEX]], i32 1
; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP10]], i32 1
; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP11]], i32 1
; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP12]], i32 1
-; UNROLL-NO-IC-NEXT: store i32 [[TMP23]], ptr [[TMP27]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
-; UNROLL-NO-IC-NEXT: store i32 [[TMP24]], ptr [[TMP28]], align 1, !alias.scope [[META20]], !noalias [[META17]]
-; UNROLL-NO-IC-NEXT: store i32 [[TMP25]], ptr [[TMP29]], align 1, !alias.scope [[META20]], !noalias [[META17]]
-; UNROLL-NO-IC-NEXT: store i32 [[TMP26]], ptr [[TMP30]], align 1, !alias.scope [[META20]], !noalias [[META17]]
+; UNROLL-NO-IC-NEXT: store i32 [[TMP24]], ptr [[TMP27]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
+; UNROLL-NO-IC-NEXT: store i32 [[TMP25]], ptr [[TMP28]], align 1, !alias.scope [[META20]], !noalias [[META17]]
+; UNROLL-NO-IC-NEXT: store i32 [[TMP26]], ptr [[TMP29]], align 1, !alias.scope [[META20]], !noalias [[META17]]
+; UNROLL-NO-IC-NEXT: store i32 [[TMP39]], ptr [[TMP30]], align 1, !alias.scope [[META20]], !noalias [[META17]]
; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)
; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -4596,7 +4596,7 @@ define void @nonprimary(ptr nocapture %a, i32 %start, i32 %i, i32 %k) {
; UNROLL: vector.body:
; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <2 x i32> [[VEC_IND]], splat (i32 2)
+; UNROLL-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i32> [[VEC_IND]], splat (i32 2)
; UNROLL-NEXT: [[TMP2:%.*]] = add nuw i32 [[I]], [[INDEX]]
; UNROLL-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP2]]
; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 2
@@ -4638,7 +4638,7 @@ define void @nonprimary(ptr nocapture %a, i32 %start, i32 %i, i32 %k) {
; UNROLL-NO-IC: vector.body:
; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <2 x i32> [[VEC_IND]], splat (i32 2)
+; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i32> [[VEC_IND]], splat (i32 2)
; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = add nuw i32 [[I]], [[INDEX]]
; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[OFFSET_IDX]]
; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 2
@@ -4680,7 +4680,7 @@ define void @nonprimary(ptr nocapture %a, i32 %start, i32 %i, i32 %k) {
; INTERLEAVE: vector.body:
; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
; INTERLEAVE-NEXT: [[TMP2:%.*]] = add nuw i32 [[I]], [[INDEX]]
; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP2]]
; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 4
@@ -5462,7 +5462,7 @@ define i64 @trunc_with_first_order_recurrence() {
; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 42>, [[VECTOR_PH]] ], [ [[STEP_ADD7:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NEXT: [[VEC_IND3:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NEXT: [[VEC_IND5:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT6:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <2 x i32> [[VEC_IND]], splat (i32 2)
+; UNROLL-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i32> [[VEC_IND]], splat (i32 2)
; UNROLL-NEXT: [[STEP_ADD7]] = add <2 x i32> [[VEC_IND3]], splat (i32 2)
; UNROLL-NEXT: [[STEP_ADD5:%.*]] = add <2 x i32> [[VEC_IND5]], splat (i32 2)
; UNROLL-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND3]], <2 x i32> <i32 1, i32 2>
@@ -5538,7 +5538,7 @@ define i64 @trunc_with_first_order_recurrence() {
; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 42>, [[VECTOR_PH]] ], [ [[STEP_ADD7:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NO-IC-NEXT: [[VEC_IND3:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NO-IC-NEXT: [[VEC_IND5:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT6:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <2 x i32> [[VEC_IND]], splat (i32 2)
+; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i32> [[VEC_IND]], splat (i32 2)
; UNROLL-NO-IC-NEXT: [[STEP_ADD7]] = add <2 x i32> [[VEC_IND3]], splat (i32 2)
; UNROLL-NO-IC-NEXT: [[STEP_ADD8:%.*]] = add <2 x i32> [[VEC_IND5]], splat (i32 2)
; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND3]], <2 x i32> <i32 1, i32 2>
@@ -5614,7 +5614,7 @@ define i64 @trunc_with_first_order_recurrence() {
; INTERLEAVE-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 42>, [[VECTOR_PH]] ], [ [[STEP_ADD7:%.*]], [[VECTOR_BODY]] ]
; INTERLEAVE-NEXT: [[VEC_IND3:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], [[VECTOR_BODY]] ]
; INTERLEAVE-NEXT: [[VEC_IND5:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT6:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
; INTERLEAVE-NEXT: [[STEP_ADD7]] = add <4 x i32> [[VEC_IND3]], splat (i32 4)
; INTERLEAVE-NEXT: [[STEP_ADD5:%.*]] = add <4 x i32> [[VEC_IND5]], splat (i32 4)
; INTERLEAVE-NEXT: [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
index 83d5615e500c9e..1b76416c3c2d19 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
@@ -46,9 +46,9 @@ define i64 @select_decreasing_induction_icmp_const_start(ptr %a) {
; IC4VF4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
; IC4VF4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; IC4VF4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
-; IC4VF4-NEXT: [[STEP_ADD:%.*]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)
-; IC4VF4-NEXT: [[STEP_ADD_2:%.*]] = add nsw <4 x i64> [[STEP_ADD]], splat (i64 -4)
-; IC4VF4-NEXT: [[STEP_ADD_3:%.*]] = add nsw <4 x i64> [[STEP_ADD_2]], splat (i64 -4)
+; IC4VF4-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
+; IC4VF4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 -4)
+; IC4VF4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 -4)
; IC4VF4-NEXT: [[OFFSET_IDX:%.*]] = sub i64 19999, [[INDEX]]
; IC4VF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
; IC4VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 -3
@@ -725,9 +725,9 @@ define i64 @select_decreasing_induction_icmp_iv_unsigned(ptr %a) {
; IC4VF4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
; IC4VF4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; IC4VF4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
-; IC4VF4-NEXT: [[STEP_ADD:%.*]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)
-; IC4VF4-NEXT: [[STEP_ADD_2:%.*]] = add nsw <4 x i64> [[STEP_ADD]], splat (i64 -4)
-; IC4VF4-NEXT: [[STEP_ADD_3:%.*]] = add nsw <4 x i64> [[STEP_ADD_2]], splat (i64 -4)
+; IC4VF4-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
+; IC4VF4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 -4)
+; IC4VF4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 -4)
; IC4VF4-NEXT: [[OFFSET_IDX:%.*]] = sub i64 9223372036854775807, [[INDEX]]
; IC4VF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
; IC4VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 -3
@@ -1195,9 +1195,9 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; IC4VF4-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
; IC4VF4-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
; IC4VF4-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP33:%.*]], %[[VECTOR_BODY]] ]
-; IC4VF4-NEXT: [[STEP_ADD:%.*]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)
-; IC4VF4-NEXT: [[STEP_ADD_2:%.*]] = add nsw <4 x i64> [[STEP_ADD]], splat (i64 -4)
-; IC4VF4-NEXT: [[STEP_ADD_3:%.*]] = add nsw <4 x i64> [[STEP_ADD_2]], splat (i64 -4)
+; IC4VF4-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
+; IC4VF4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 -4)
+; IC4VF4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 -4)
; IC4VF4-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
; IC4VF4-NEXT: [[TMP4:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
; IC4VF4-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
index 48fa114ea206b7..a5d6164d92101e 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
@@ -92,9 +92,9 @@ define i64 @select_non_const_iv_start_signed_guard(ptr %a, i64 %rdx_start, i64 %
; CHECK-VF4IC4-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add nsw <4 x i64> [[VEC_IND1]], splat (i64 4)
-; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
-; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = add nsw <4 x i64> [[STEP_ADD_2]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 4)
; CHECK-VF4IC4-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[IV_START]], [[INDEX]]
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i64 4
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
index afceae5122a035..7d09b11cc991f9 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
@@ -1477,9 +1477,9 @@ define i64 @select_icmp_min_valid_iv_start(ptr %a, ptr %b, i64 %rdx.start, i64 %
; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
-; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add nsw <4 x i64> [[STEP_ADD_2]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 4)
; CHECK-VF4IC4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 4
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 8
@@ -1702,9 +1702,9 @@ define i64 @select_icmp_unsigned_iv_range(ptr %a, ptr %b, i64 %rdx.start) {
; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
-; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add nsw <4 x i64> [[STEP_ADD_2]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 4)
; CHECK-VF4IC4-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 4
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 8
@@ -2140,9 +2140,9 @@ define i64 @not_vectorized_select_icmp_iv_out_of_bound(ptr %a, ptr %b, i64 %rdx.
; CHECK-VF4IC4-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add nsw <4 x i64> [[VEC_IND1]], splat (i64 4)
-; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
-; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = add nsw <4 x i64> [[STEP_ADD_2]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 4)
; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-VF4IC4-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 4
; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 8
@@ -2409,9 +2409,9 @@ define i64 @not_vectorized_select_icmp_non_const_iv_start_value(ptr %a, ptr %b,
; CHECK-VF4IC4-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add nuw nsw <4 x i64> [[VEC_IND1]], splat (i64 4)
-; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add nuw nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
-; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = add nuw nsw <4 x i64> [[STEP_ADD_2]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND1]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add nuw <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = add nuw <4 x i64> [[STEP_ADD_2]], splat (i64 4)
; CHECK-VF4IC4-NEXT: [[OFFSET_IDX:%.*]] = add nuw i64 [[IVSTART]], [[INDEX]]
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i64 4
diff --git a/llvm/test/Transforms/LoopVectorize/loop-form.ll b/llvm/test/Transforms/LoopVectorize/loop-form.ll
index 079fde5a8065e9..b6ef0f229bbd2c 100644
--- a/llvm/test/Transforms/LoopVectorize/loop-form.ll
+++ b/llvm/test/Transforms/LoopVectorize/loop-form.ll
@@ -75,7 +75,7 @@ define void @bottom_tested(ptr %p, i32 %n) {
; TAILFOLD-NEXT: br label [[PRED_STORE_CONTINUE2]]
; TAILFOLD: pred.store.continue2:
; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
; TAILFOLD-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; TAILFOLD: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll
index fc6cfe5942254a..02c88abac93469 100644
--- a/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll
@@ -62,7 +62,7 @@ define void @maxvf3() {
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE6]]
; CHECK: pred.store.continue6:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll
index 294dbbbb3a5e26..6c49430ae60e2c 100644
--- a/llvm/test/Transforms/LoopVectorize/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -486,7 +486,7 @@ define i32 @pr45526() optsize {
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; CHECK-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -511,7 +511,7 @@ define i32 @pr45526() optsize {
; PGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; PGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; PGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; PGSO-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; PGSO-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512
; PGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; PGSO: [[MIDDLE_BLOCK]]:
@@ -536,7 +536,7 @@ define i32 @pr45526() optsize {
; NPGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; NPGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; NPGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; NPGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; NPGSO-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; NPGSO-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512
; NPGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; NPGSO: [[MIDDLE_BLOCK]]:
@@ -577,7 +577,7 @@ define i32 @pr45526_pgso() !prof !14 {
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; CHECK-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -602,7 +602,7 @@ define i32 @pr45526_pgso() !prof !14 {
; PGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; PGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; PGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; PGSO-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; PGSO-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512
; PGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; PGSO: [[MIDDLE_BLOCK]]:
@@ -688,7 +688,7 @@ define void @stride1(ptr noalias %B, i32 %BStride) optsize {
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; CHECK: [[PRED_STORE_CONTINUE2]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -726,7 +726,7 @@ define void @stride1(ptr noalias %B, i32 %BStride) optsize {
; PGSO-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; PGSO: [[PRED_STORE_CONTINUE2]]:
; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; PGSO-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; PGSO-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
; PGSO-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026
; PGSO-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; PGSO: [[MIDDLE_BLOCK]]:
@@ -764,7 +764,7 @@ define void @stride1(ptr noalias %B, i32 %BStride) optsize {
; NPGSO-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; NPGSO: [[PRED_STORE_CONTINUE2]]:
; NPGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; NPGSO-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; NPGSO-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
; NPGSO-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026
; NPGSO-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; NPGSO: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll b/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
index d37414ad69a5a6..ad731f3c8808af 100644
--- a/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
@@ -21,8 +21,8 @@ define void @test(ptr %base_a, ptr %base_b, i32 %ntypes) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT2]]
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[N]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = mul nuw <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[N]], 2
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -35,7 +35,7 @@ define void @test(ptr %base_a, ptr %base_b, i32 %ntypes) {
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [8 x i8], ptr [[BASE_B]], i64 [[TMP20]]
; CHECK-NEXT: call void @llvm.masked.store.v4p0.p0(<4 x ptr> [[TMP3]], ptr align 8 [[TMP23]], <4 x i1> [[TMP0]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP20]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND1]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND1]], [[BROADCAST_SPLAT4]]
; CHECK-NEXT: [[VEC_IND_NEXT6]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-assume.ll b/llvm/test/Transforms/LoopVectorize/scalable-assume.ll
index 24c46fa93626a5..01b12864504aac 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-assume.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-assume.ll
@@ -166,7 +166,7 @@ define void @predicated_assume(ptr noalias nocapture readonly %a, ptr noalias no
; CHECK-NEXT: store <vscale x 2 x float> [[TMP15]], ptr [[TMP17]], align 4
; CHECK-NEXT: store <vscale x 2 x float> [[TMP16]], ptr [[TMP20]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll b/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll
index 5bd4b151c75d53..e581a022d5b536 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll
@@ -37,7 +37,7 @@ define void @add_ind64_unrolled(ptr noalias nocapture %a, ptr noalias nocapture
; CHECK-NEXT: store <vscale x 2 x i64> [[TMP12]], ptr [[TMP14]], align 8
; CHECK-NEXT: store <vscale x 2 x i64> [[TMP13]], ptr [[TMP16]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[STEP_ADD]], [[DOTSPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[STEP_ADD]], [[DOTSPLAT]]
; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
@@ -109,7 +109,7 @@ define void @add_ind64_unrolled_nxv1i64(ptr noalias nocapture %a, ptr noalias no
; CHECK-NEXT: store <vscale x 1 x i64> [[TMP11]], ptr [[TMP13]], align 8
; CHECK-NEXT: store <vscale x 1 x i64> [[TMP12]], ptr [[TMP15]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 1 x i64> [[STEP_ADD]], [[DOTSPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 1 x i64> [[STEP_ADD]], [[DOTSPLAT]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: middle.block:
@@ -172,9 +172,9 @@ define void @add_unique_ind32(ptr noalias nocapture %a, i64 %n) {
; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[DOTCAST]], 1
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; CHECK-NEXT: [[TMP7:%.*]] = mul nuw nsw <vscale x 4 x i32> [[TMP6]], splat (i32 2)
+; CHECK-NEXT: [[TMP7:%.*]] = mul nuw <vscale x 4 x i32> [[TMP6]], splat (i32 2)
; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP1]] to i32
-; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i32 [[TMP9]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = shl nuw i32 [[TMP9]], 1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP8]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -184,7 +184,7 @@ define void @add_unique_ind32(ptr noalias nocapture %a, i64 %n) {
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: store <vscale x 4 x i32> [[VEC_IND]], ptr [[TMP10]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/struct-return.ll b/llvm/test/Transforms/LoopVectorize/struct-return.ll
index b236885028fc46..1719eb6428dcc5 100644
--- a/llvm/test/Transforms/LoopVectorize/struct-return.ll
+++ b/llvm/test/Transforms/LoopVectorize/struct-return.ll
@@ -909,7 +909,7 @@ define float @struct_return_tail_folded_predicated_for_and_live_outs(ptr noalias
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; CHECK: [[PRED_STORE_CONTINUE4]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-argmin-reduction.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-argmin-reduction.ll
index 3ddb1316b05b4e..5d86cfd9ce56d6 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-argmin-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-argmin-reduction.ll
@@ -36,7 +36,7 @@ define i64 @argmin_first_index(ptr %arr, i64 %n, i64 %start) {
; DATA-NEXT: [[TMP8:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP5]], <vscale x 2 x i1> zeroinitializer
; DATA-NEXT: [[TMP9]] = select <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x i64> [[VEC_PHI3]]
; DATA-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
-; DATA-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; DATA-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; DATA-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; DATA-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; DATA: [[MIDDLE_BLOCK]]:
@@ -133,7 +133,7 @@ define i64 @argmin_first_index(ptr %arr, i64 %n, i64 %start) {
; DATA_AND_CONTROL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP18]], i64 [[TRIP_COUNT_MINUS_VF]])
; DATA_AND_CONTROL-NEXT: [[TMP28:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; DATA_AND_CONTROL-NEXT: [[TMP29:%.*]] = xor i1 [[TMP28]], true
-; DATA_AND_CONTROL-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; DATA_AND_CONTROL-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; DATA_AND_CONTROL-NEXT: [[INDEX_NEXT]] = add i64 [[TMP18]], [[TMP1]]
; DATA_AND_CONTROL-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_LOAD_CONTINUE5]], !llvm.loop [[LOOP0:![0-9]+]]
; DATA_AND_CONTROL: [[MIDDLE_BLOCK]]:
@@ -202,7 +202,7 @@ define i64 @argmin_last_index(ptr %arr, i64 %n, i64 %start) {
; DATA-NEXT: [[TMP8:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP5]], <vscale x 2 x i1> zeroinitializer
; DATA-NEXT: [[TMP9]] = select <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x i64> [[VEC_PHI3]]
; DATA-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
-; DATA-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; DATA-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; DATA-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; DATA-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; DATA: [[MIDDLE_BLOCK]]:
@@ -299,7 +299,7 @@ define i64 @argmin_last_index(ptr %arr, i64 %n, i64 %start) {
; DATA_AND_CONTROL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP18]], i64 [[TRIP_COUNT_MINUS_VF]])
; DATA_AND_CONTROL-NEXT: [[TMP28:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; DATA_AND_CONTROL-NEXT: [[TMP29:%.*]] = xor i1 [[TMP28]], true
-; DATA_AND_CONTROL-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; DATA_AND_CONTROL-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; DATA_AND_CONTROL-NEXT: [[INDEX_NEXT]] = add i64 [[TMP18]], [[TMP1]]
; DATA_AND_CONTROL-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_LOAD_CONTINUE5]], !llvm.loop [[LOOP3:![0-9]+]]
; DATA_AND_CONTROL: [[MIDDLE_BLOCK]]:
@@ -337,3 +337,19 @@ for.end:
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.vectorize.enable"}
+;.
+; DATA: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; DATA: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; DATA: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; DATA: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+;.
+; DATA_NO_LANEMASK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; DATA_NO_LANEMASK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; DATA_NO_LANEMASK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; DATA_NO_LANEMASK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+;.
+; DATA_AND_CONTROL: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; DATA_AND_CONTROL: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; DATA_AND_CONTROL: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; DATA_AND_CONTROL: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-masked-mem-opts.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-masked-mem-opts.ll
index 04dc91f62712db..4c59c9ecbdcbb1 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-masked-mem-opts.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-masked-mem-opts.ll
@@ -139,7 +139,7 @@ define void @non_consecutive_copy(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; CHECK: [[PRED_STORE_CONTINUE3]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX1]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
index fc32a9179d941f..18c92dce0377d3 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
@@ -145,7 +145,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) v
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL: [[SCALAR_PH]]:
@@ -399,7 +399,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
@@ -615,7 +615,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_GATHER]], ptr align 8 [[TMP4]], <4 x i1> [[TMP2]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[MIDDLE_BLOCK]], !llvm.loop [[LOOP4:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL: [[SCALAR_PH]]:
@@ -860,7 +860,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL: [[SCALAR_PH]]:
@@ -1133,7 +1133,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_GATHER]], ptr align 8 [[TMP4]], <4 x i1> [[TMP2]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[MIDDLE_BLOCK]], !llvm.loop [[LOOP6:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL: [[SCALAR_PH]]:
@@ -1576,7 +1576,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP47]], ptr align 8 [[TMP7]], <4 x i1> [[TMP3]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
More information about the flang-commits
mailing list