[llvm] [TTI/RISCV] Strip getMaxVScale (PR #220547)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 2 09:05:14 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/220547
>From 4e9bbd4d187b39de79949ebdbbdd6623f3c1a7c6 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 2 Sep 2026 11:41:20 +0100
Subject: [PATCH] [TTI/RISCV] Strip getMaxVScale
RISC-V is the only target using it at the moment, and it seems to be
unnecessary: should be subsumed by vscale_range.
---
.../llvm/Analysis/TargetTransformInfo.h | 4 -
llvm/lib/Analysis/TargetTransformInfo.cpp | 4 -
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 6 -
.../Target/RISCV/RISCVTargetTransformInfo.h | 1 -
.../Vectorize/LoopVectorizationPlanner.cpp | 10 +-
.../Vectorize/LoopVectorizationPlanner.h | 3 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 7 +-
.../RISCV/blocks-with-dead-instructions.ll | 2 +-
.../LoopVectorize/RISCV/dead-ops-cost.ll | 16 +-
.../LoopVectorize/RISCV/defaults.ll | 6 +-
.../Transforms/LoopVectorize/RISCV/divrem.ll | 20 +-
.../first-order-recurrence-scalable-vf1.ll | 4 +-
.../LoopVectorize/RISCV/fminimumnum.ll | 181 +++++------
.../RISCV/gather-scatter-cost.ll | 7 +-
.../LoopVectorize/RISCV/induction-costs.ll | 4 +-
.../indvar-overflow-check-scalable-tc.ll | 6 +-
.../LoopVectorize/RISCV/inloop-reduction.ll | 16 +-
.../RISCV/interleaved-accesses.ll | 22 +-
.../RISCV/interleaved-masked-access.ll | 10 +-
.../Transforms/LoopVectorize/RISCV/lmul.ll | 33 +-
.../LoopVectorize/RISCV/mask-index-type.ll | 2 +-
.../RISCV/masked_gather_scatter.ll | 2 +-
.../RISCV/partial-reduce-dot-product.ll | 32 +-
...ruction-or-drop-poison-generating-flags.ll | 2 +-
.../LoopVectorize/RISCV/predicated-costs.ll | 4 +-
.../LoopVectorize/RISCV/reg-usage-prune-vf.ll | 2 +-
.../RISCV/riscv-vector-reverse.ll | 34 +-
.../runtime-check-dependent-on-stride.ll | 6 +-
.../LoopVectorize/RISCV/safe-dep-distance.ll | 10 +-
.../LoopVectorize/RISCV/scalable-basics.ll | 24 +-
.../LoopVectorize/RISCV/scalable-tailfold.ll | 14 +-
.../LoopVectorize/RISCV/scalable-vf-hint.ll | 33 +-
.../RISCV/strided-access-wide-stride.ll | 4 +-
.../RISCV/strided-accesses-i64-rv32.ll | 2 +-
.../RISCV/strided-accesses-narrow-iv.ll | 130 ++------
.../LoopVectorize/RISCV/strided-accesses.ll | 61 ++--
.../RISCV/tail-folding-bin-unary-ops-args.ll | 293 +++++++++---------
.../LoopVectorize/RISCV/tail-folding-div.ll | 72 ++---
.../RISCV/tail-folding-interleave.ll | 37 +--
.../RISCV/tail-folding-reverse-load-store.ll | 15 +-
.../RISCV/tail-folding-safe-dep-distance.ll | 20 +-
...sform-narrow-interleave-to-widen-memory.ll | 8 +-
.../truncate-to-minimal-bitwidth-cost.ll | 14 +-
.../truncate-to-minimal-bitwidth-evl-crash.ll | 37 +--
.../LoopVectorize/RISCV/uniform-load-store.ll | 80 ++---
.../LoopVectorize/RISCV/zvl1024b-crash.ll | 2 +-
.../VPlan/RISCV/partial-reduce-dot-product.ll | 2 +-
.../VPlan/RISCV/reductions-evl.ll | 30 +-
.../VPlan/RISCV/vplan-vp-intrinsics.ll | 231 ++++++++------
.../RISCV/any-of-vectorization.ll | 2 +-
.../PhaseOrdering/loop-vectorize-bfi.ll | 2 +-
51 files changed, 706 insertions(+), 863 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 5e22b890ec975..6f620366f73b8 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1366,10 +1366,6 @@ class TargetTransformInfo {
/// \return The width of the smallest vector register type.
LLVM_ABI unsigned getMinVectorRegisterBitWidth() const;
- /// \return The maximum value of vscale if the target specifies an
- /// architectural maximum vector length, and std::nullopt otherwise.
- LLVM_ABI std::optional<unsigned> getMaxVScale() const;
-
/// \return the value of vscale to tune the cost model for.
LLVM_ABI std::optional<unsigned> getVScaleForTuning() const;
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 67dee54028c0e..71f55f2e7e046 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -839,10 +839,6 @@ unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {
return TTIImpl->getMinVectorRegisterBitWidth();
}
-std::optional<unsigned> TargetTransformInfo::getMaxVScale() const {
- return TTIImpl->getMaxVScale();
-}
-
std::optional<unsigned> TargetTransformInfo::getVScaleForTuning() const {
return TTIImpl->getVScaleForTuning();
}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index cd14ff3d2dded..9538697368a89 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -444,12 +444,6 @@ bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
}
}
-std::optional<unsigned> RISCVTTIImpl::getMaxVScale() const {
- if (ST->hasVInstructions())
- return ST->getRealMaxVLen() / RISCV::RVVBitsPerBlock;
- return BaseT::getMaxVScale();
-}
-
std::optional<unsigned> RISCVTTIImpl::getVScaleForTuning() const {
if (ST->hasVInstructions())
if (unsigned MinVLen = ST->getRealMinVLen();
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 447c3d7c3326e..4523ebd6ae892 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -128,7 +128,6 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
return ST->hasVInstructions() ? TailFoldingStyle::DataWithEVL
: TailFoldingStyle::None;
}
- std::optional<unsigned> getMaxVScale() const override;
std::optional<unsigned> getVScaleForTuning() const override;
TypeSize
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index c464c7894ae5c..31d3f49e167f0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -301,11 +301,7 @@ ElementCount VFSelectionContext::getMaximizedVFForTarget(
return MaxVF;
}
-std::optional<unsigned> llvm::getMaxVScale(const Function &F,
- const TargetTransformInfo &TTI) {
- if (std::optional<unsigned> MaxVScale = TTI.getMaxVScale())
- return MaxVScale;
-
+std::optional<unsigned> llvm::getMaxVScale(const Function &F) {
if (F.hasFnAttribute(Attribute::VScaleRange))
return F.getFnAttribute(Attribute::VScaleRange).getVScaleRangeMax();
@@ -359,7 +355,7 @@ bool VFSelectionContext::isScalableVectorizationAllowed() {
return false;
}
- if (!Legal->isSafeForAnyVectorWidth() && !getMaxVScale(F, TTI)) {
+ if (!Legal->isSafeForAnyVectorWidth() && !getMaxVScale(F)) {
reportVectorizationInfo("The target does not provide maximum vscale value "
"for safe distance analysis.",
"ScalableVFUnfeasible", ORE, TheLoop);
@@ -380,7 +376,7 @@ VFSelectionContext::getMaxLegalScalableVF(unsigned MaxSafeElements) {
if (Legal->isSafeForAnyVectorWidth())
return MaxScalableVF;
- std::optional<unsigned> MaxVScale = getMaxVScale(F, TTI);
+ std::optional<unsigned> MaxVScale = getMaxVScale(F);
// Limit MaxScalableVF by the maximum safe dependence distance.
MaxScalableVF = ElementCount::getScalable(MaxSafeElements / *MaxVScale);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 2e5b0f64bfed2..da0616d371f2c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -56,8 +56,7 @@ extern cl::opt<bool> PreferInLoopReductions;
/// \return An upper bound for vscale based on TTI or the vscale_range
/// attribute.
-std::optional<unsigned> getMaxVScale(const Function &F,
- const TargetTransformInfo &TTI);
+std::optional<unsigned> getMaxVScale(const Function &F);
// Utility functions that are used by different vectorization classes
namespace LoopVectorizationUtils {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ba0421869216f..22aa221e4f652 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1927,8 +1927,7 @@ static bool isIndvarOverflowCheckKnownFalse(
uint64_t MaxVF = VF.getKnownMinValue();
uint64_t MaxTC = TC->getKnownMinValue();
if (VF.isScalable() || TC->isScalable()) {
- std::optional<unsigned> MaxVScale =
- getMaxVScale(*Cost->TheFunction, Cost->TTI);
+ std::optional<unsigned> MaxVScale = getMaxVScale(*Cost->TheFunction);
if (!MaxVScale)
return false;
if (VF.isScalable())
@@ -3002,7 +3001,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
std::optional<unsigned> MaxPowerOf2RuntimeVF =
MaxFactors.FixedVF.getFixedValue();
if (MaxFactors.ScalableVF) {
- std::optional<unsigned> MaxVScale = getMaxVScale(*TheFunction, TTI);
+ std::optional<unsigned> MaxVScale = getMaxVScale(*TheFunction);
if (MaxVScale) {
MaxPowerOf2RuntimeVF = std::max<unsigned>(
*MaxPowerOf2RuntimeVF,
@@ -5944,7 +5943,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
RUN_VPLAN_PASS(VPlanTransforms::materializeBackedgeTakenCount, BestVPlan,
VectorPH);
std::optional<uint64_t> MaxRuntimeStep;
- if (auto MaxVScale = getMaxVScale(*OrigLoop->getHeader()->getParent(), TTI))
+ if (auto MaxVScale = getMaxVScale(*OrigLoop->getHeader()->getParent()))
MaxRuntimeStep = uint64_t(*MaxVScale) * BestVF.getKnownMinValue() * BestUF;
assert((LI->getUniqueLatchExitBlock(*OrigLoop) || RequiresScalarEpilogue) &&
"loops not exiting via the latch without required epilogue?");
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
index 333f20a0d8d06..024dc163b377b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
@@ -539,7 +539,7 @@ exit:
ret void
}
-attributes #0 = { "target-features"="+64bit,+v" }
+attributes #0 = { "target-features"="+64bit,+v" vscale_range(4, 1024) }
;.
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 23d9b5c5b7c52..139498a0b3b26 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -5,7 +5,7 @@ target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
; Test with a dead load in the loop, from
; https://github.com/llvm/llvm-project/issues/99701
-define void @dead_load(ptr %p, i16 %start) {
+define void @dead_load(ptr %p, i16 %start) vscale_range(4, 1024) {
; CHECK-LABEL: define void @dead_load(
; CHECK-SAME: ptr [[P:%.*]], i16 [[START:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -58,7 +58,7 @@ exit:
; Loop with a live-out %l and scalar epilogue required due to an interleave
; group. As the scalar epilogue is required the live-out is fed from the scalar
; epilogue and dead in the vector loop.
-define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
+define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) vscale_range(4, 1024) {
; CHECK-LABEL: define i8 @dead_live_out_due_to_scalar_epilogue_required(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -128,7 +128,7 @@ exit:
; Test case for https://github.com/llvm/llvm-project/issues/106780.
-define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0 {
+define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) vscale_range(2, 1024) {
; CHECK-LABEL: define i32 @cost_of_exit_branch_and_cond_insts(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i1 [[C:%.*]], i16 [[X:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -267,9 +267,9 @@ return:
}
; Test case for https://github.com/llvm/llvm-project/issues/107473.
-define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
+define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) vscale_range(2, 1024) {
; CHECK-LABEL: define void @test_phi_in_latch_redundant(
-; CHECK-SAME: ptr [[DST:%.*]], i32 [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[A:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -318,9 +318,9 @@ exit:
}
; Test for https://github.com/llvm/llvm-project/issues/108098.
-define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %src, ptr noalias %dst) #0 {
+define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %src, ptr noalias %dst) vscale_range(4, 1024) {
; CHECK-LABEL: define void @gather_interleave_group_with_dead_insert_pos(
-; CHECK-SAME: i64 [[N:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: i64 [[N:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
; CHECK-NEXT: [[TMP0:%.*]] = add nuw i64 [[SMAX]], 1
@@ -377,8 +377,6 @@ exit:
ret void
}
-attributes #0 = { "target-features"="+64bit,+v" }
-
;.
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll b/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll
index 3fd90b2848848..87143af37e76b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll
@@ -10,7 +10,7 @@
target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
target triple = "riscv64"
-define void @vector_add(ptr noalias nocapture %a, i64 %v) {
+define void @vector_add(ptr noalias nocapture %a, i64 %v) vscale_range(4, 1024) {
; CHECK-LABEL: @vector_add(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -53,7 +53,7 @@ for.end:
ret void
}
-define i64 @vector_add_reduce(ptr noalias nocapture %a) {
+define i64 @vector_add_reduce(ptr noalias nocapture %a) vscale_range(4, 1024) {
; CHECK-LABEL: @vector_add_reduce(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -72,7 +72,7 @@ define i64 @vector_add_reduce(ptr noalias nocapture %a) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP9]])
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index d02dc36f7a6ae..8bbf90793da01 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -7,7 +7,7 @@
target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
target triple = "riscv64"
-define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
+define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @vector_udiv(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -71,7 +71,7 @@ for.end:
ret void
}
-define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
+define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @vector_sdiv(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -135,7 +135,7 @@ for.end:
ret void
}
-define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) {
+define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @vector_urem(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -199,7 +199,7 @@ for.end:
ret void
}
-define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) {
+define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @vector_srem(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -263,7 +263,7 @@ for.end:
ret void
}
-define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
+define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @predicated_udiv(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -339,7 +339,7 @@ for.end:
ret void
}
-define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
+define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @predicated_sdiv(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -415,7 +415,7 @@ for.end:
ret void
}
-define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {
+define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @predicated_udiv_by_constant(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -485,7 +485,7 @@ for.end:
ret void
}
-define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {
+define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @predicated_sdiv_by_constant(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -555,7 +555,7 @@ for.end:
ret void
}
-define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {
+define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @predicated_sdiv_by_minus_one(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -628,7 +628,7 @@ for.end:
; Test for https://github.com/llvm/llvm-project/issues/159402. For invariant divisors,
; selects can be introduced outside the vector loop and their cost should not be
; considered for each loop iteration.
-define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
+define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @udiv_sdiv_with_invariant_divisors(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll b/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
index 5fe69bb581e47..1a62621fc92e2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
@@ -19,7 +19,7 @@ define i64 @pr97452_scalable_vf1_for(ptr %src, ptr noalias %dst) #0 {
; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 [[TMP5]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[VP_OP_LOAD:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 23, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[PREV_EVL:%.*]] = phi i32 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
@@ -30,7 +30,7 @@ define i64 @pr97452_scalable_vf1_for(ptr %src, ptr noalias %dst) #0 {
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[EVL_BASED_IV]]
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP10]], ptr align 8 [[TMP11]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP6]])
; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP6]] to i64
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP12]], [[EVL_BASED_IV]]
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP12]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP12]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll b/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
index 449d260d878bd..b6050309e4cc1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
@@ -1,24 +1,19 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; FIXME: fmaximumnum/fminimumnum have no vectorizing support yet.
; RUN: opt --passes=loop-vectorize --mtriple=riscv64 -mattr="+v,+zvfh" -S < %s | FileCheck %s
; RUN: opt --passes=loop-vectorize --mtriple=riscv64 -mattr="+v,+zvfhmin" -S < %s | FileCheck %s --check-prefix=ZVFHMIN
-define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) {
+define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) vscale_range(2, 1024) {
; CHECK-LABEL: define void @fmin32(
; CHECK-SAME: ptr noundef readonly captures(none) [[INPUT1:%.*]], ptr noundef readonly captures(none) [[INPUT2:%.*]], ptr noundef writeonly captures(none) [[OUTPUT:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[INPUT23:%.*]] = ptrtoaddr ptr [[INPUT2]] to i64
; CHECK-NEXT: [[INPUT12:%.*]] = ptrtoaddr ptr [[INPUT1]] to i64
; CHECK-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 2
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 17)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP15]], 4
-; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP4]], -1
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[TMP3]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP12]]
@@ -26,8 +21,10 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP1]], 1
; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP11]], [[TMP12]]
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP7]], 2
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -44,10 +41,9 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[INPUT1]], i64 0, i64 [[INDVARS_IV]]
; CHECK-NEXT: [[IN1:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[INPUT2]], i64 0, i64 [[INDVARS_IV]]
@@ -63,19 +59,15 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
;
; ZVFHMIN-LABEL: define void @fmin32(
; ZVFHMIN-SAME: ptr noundef readonly captures(none) [[INPUT1:%.*]], ptr noundef readonly captures(none) [[INPUT2:%.*]], ptr noundef writeonly captures(none) [[OUTPUT:%.*]]) #[[ATTR0:[0-9]+]] {
-; ZVFHMIN-NEXT: [[ENTRY:.*]]:
+; ZVFHMIN-NEXT: [[ENTRY:.*:]]
; ZVFHMIN-NEXT: [[INPUT23:%.*]] = ptrtoaddr ptr [[INPUT2]] to i64
; ZVFHMIN-NEXT: [[INPUT12:%.*]] = ptrtoaddr ptr [[INPUT1]] to i64
; ZVFHMIN-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
-; ZVFHMIN-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 17)
-; ZVFHMIN-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
-; ZVFHMIN-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; ZVFHMIN-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
-; ZVFHMIN-NEXT: [[TMP14:%.*]] = add i64 [[TMP4]], -1
+; ZVFHMIN-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP14:%.*]] = add nsw i64 [[TMP2]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP12:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
@@ -83,8 +75,10 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[TMP16:%.*]] = sub i64 [[TMP8]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP16]], [[TMP14]]
; ZVFHMIN-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; ZVFHMIN: [[VECTOR_PH]]:
+; ZVFHMIN-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 2
; ZVFHMIN-NEXT: br label %[[VECTOR_BODY:.*]]
; ZVFHMIN: [[VECTOR_BODY]]:
; ZVFHMIN-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -101,10 +95,9 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN: [[MIDDLE_BLOCK]]:
; ZVFHMIN-NEXT: br label %[[EXIT:.*]]
; ZVFHMIN: [[SCALAR_PH]]:
-; ZVFHMIN-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; ZVFHMIN-NEXT: br label %[[FOR_BODY:.*]]
; ZVFHMIN: [[FOR_BODY]]:
-; ZVFHMIN-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; ZVFHMIN-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
; ZVFHMIN-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[INPUT1]], i64 0, i64 [[IV]]
; ZVFHMIN-NEXT: [[IN1:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; ZVFHMIN-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[INPUT2]], i64 0, i64 [[IV]]
@@ -139,22 +132,18 @@ exit:
}
-define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) {
+define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) vscale_range(2, 1024) {
; CHECK-LABEL: define void @fmax32(
; CHECK-SAME: ptr noundef readonly captures(none) [[INPUT1:%.*]], ptr noundef readonly captures(none) [[INPUT2:%.*]], ptr noundef writeonly captures(none) [[OUTPUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[INPUT23:%.*]] = ptrtoaddr ptr [[INPUT2]] to i64
; CHECK-NEXT: [[INPUT12:%.*]] = ptrtoaddr ptr [[INPUT1]] to i64
; CHECK-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 2
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 17)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP15]], 4
-; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP4]], -1
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[TMP3]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP12]]
@@ -162,8 +151,10 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP1]], 1
; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP11]], [[TMP12]]
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP7]], 2
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -180,10 +171,9 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[INPUT1]], i64 0, i64 [[INDVARS_IV]]
; CHECK-NEXT: [[IN1:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[INPUT2]], i64 0, i64 [[INDVARS_IV]]
@@ -199,19 +189,15 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
;
; ZVFHMIN-LABEL: define void @fmax32(
; ZVFHMIN-SAME: ptr noundef readonly captures(none) [[INPUT1:%.*]], ptr noundef readonly captures(none) [[INPUT2:%.*]], ptr noundef writeonly captures(none) [[OUTPUT:%.*]]) #[[ATTR0]] {
-; ZVFHMIN-NEXT: [[ENTRY:.*]]:
+; ZVFHMIN-NEXT: [[ENTRY:.*:]]
; ZVFHMIN-NEXT: [[INPUT23:%.*]] = ptrtoaddr ptr [[INPUT2]] to i64
; ZVFHMIN-NEXT: [[INPUT12:%.*]] = ptrtoaddr ptr [[INPUT1]] to i64
; ZVFHMIN-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
-; ZVFHMIN-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 17)
-; ZVFHMIN-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
-; ZVFHMIN-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; ZVFHMIN-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
-; ZVFHMIN-NEXT: [[TMP14:%.*]] = add i64 [[TMP4]], -1
+; ZVFHMIN-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP14:%.*]] = add nsw i64 [[TMP2]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP12:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
@@ -219,8 +205,10 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[TMP16:%.*]] = sub i64 [[TMP8]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP16]], [[TMP14]]
; ZVFHMIN-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; ZVFHMIN: [[VECTOR_PH]]:
+; ZVFHMIN-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 2
; ZVFHMIN-NEXT: br label %[[VECTOR_BODY:.*]]
; ZVFHMIN: [[VECTOR_BODY]]:
; ZVFHMIN-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -237,10 +225,9 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN: [[MIDDLE_BLOCK]]:
; ZVFHMIN-NEXT: br label %[[EXIT:.*]]
; ZVFHMIN: [[SCALAR_PH]]:
-; ZVFHMIN-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; ZVFHMIN-NEXT: br label %[[FOR_BODY:.*]]
; ZVFHMIN: [[FOR_BODY]]:
-; ZVFHMIN-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; ZVFHMIN-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
; ZVFHMIN-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[INPUT1]], i64 0, i64 [[IV]]
; ZVFHMIN-NEXT: [[IN1:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; ZVFHMIN-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[INPUT2]], i64 0, i64 [[IV]]
@@ -275,22 +262,18 @@ exit:
}
-define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) {
+define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) vscale_range(2, 1024) {
; CHECK-LABEL: define void @fmin64(
; CHECK-SAME: ptr noundef readonly captures(none) [[INPUT1:%.*]], ptr noundef readonly captures(none) [[INPUT2:%.*]], ptr noundef writeonly captures(none) [[OUTPUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[INPUT23:%.*]] = ptrtoaddr ptr [[INPUT2]] to i64
; CHECK-NEXT: [[INPUT12:%.*]] = ptrtoaddr ptr [[INPUT1]] to i64
; CHECK-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 17)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP15]], 4
-; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP4]], -1
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[TMP3]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP12]]
@@ -298,8 +281,10 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP1]], 1
; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP11]], [[TMP12]]
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP7]], 1
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -316,10 +301,9 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[INPUT1]], i64 0, i64 [[INDVARS_IV]]
; CHECK-NEXT: [[IN1:%.*]] = load double, ptr [[ARRAYIDX]], align 8
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[INPUT2]], i64 0, i64 [[INDVARS_IV]]
@@ -335,19 +319,15 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
;
; ZVFHMIN-LABEL: define void @fmin64(
; ZVFHMIN-SAME: ptr noundef readonly captures(none) [[INPUT1:%.*]], ptr noundef readonly captures(none) [[INPUT2:%.*]], ptr noundef writeonly captures(none) [[OUTPUT:%.*]]) #[[ATTR0]] {
-; ZVFHMIN-NEXT: [[ENTRY:.*]]:
+; ZVFHMIN-NEXT: [[ENTRY:.*:]]
; ZVFHMIN-NEXT: [[INPUT23:%.*]] = ptrtoaddr ptr [[INPUT2]] to i64
; ZVFHMIN-NEXT: [[INPUT12:%.*]] = ptrtoaddr ptr [[INPUT1]] to i64
; ZVFHMIN-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
-; ZVFHMIN-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 17)
-; ZVFHMIN-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
-; ZVFHMIN-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; ZVFHMIN-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
-; ZVFHMIN-NEXT: [[TMP14:%.*]] = add i64 [[TMP4]], -1
+; ZVFHMIN-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP14:%.*]] = add nsw i64 [[TMP2]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP12:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
@@ -355,8 +335,10 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[TMP16:%.*]] = sub i64 [[TMP8]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP16]], [[TMP14]]
; ZVFHMIN-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; ZVFHMIN: [[VECTOR_PH]]:
+; ZVFHMIN-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 1
; ZVFHMIN-NEXT: br label %[[VECTOR_BODY:.*]]
; ZVFHMIN: [[VECTOR_BODY]]:
; ZVFHMIN-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -373,10 +355,9 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN: [[MIDDLE_BLOCK]]:
; ZVFHMIN-NEXT: br label %[[EXIT:.*]]
; ZVFHMIN: [[SCALAR_PH]]:
-; ZVFHMIN-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; ZVFHMIN-NEXT: br label %[[FOR_BODY:.*]]
; ZVFHMIN: [[FOR_BODY]]:
-; ZVFHMIN-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; ZVFHMIN-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
; ZVFHMIN-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[INPUT1]], i64 0, i64 [[IV]]
; ZVFHMIN-NEXT: [[IN1:%.*]] = load double, ptr [[ARRAYIDX]], align 8
; ZVFHMIN-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[INPUT2]], i64 0, i64 [[IV]]
@@ -411,22 +392,18 @@ exit:
}
-define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) {
+define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) vscale_range(2, 1024) {
; CHECK-LABEL: define void @fmax64(
; CHECK-SAME: ptr noundef readonly captures(none) [[INPUT1:%.*]], ptr noundef readonly captures(none) [[INPUT2:%.*]], ptr noundef writeonly captures(none) [[OUTPUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[INPUT23:%.*]] = ptrtoaddr ptr [[INPUT2]] to i64
; CHECK-NEXT: [[INPUT12:%.*]] = ptrtoaddr ptr [[INPUT1]] to i64
; CHECK-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP13]], i64 17)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[TMP15]], 4
-; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP4]], -1
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[TMP3]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP12]]
@@ -434,8 +411,10 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP1]], 1
; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP11]], [[TMP12]]
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP7]], 1
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -452,10 +431,9 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[INPUT1]], i64 0, i64 [[INDVARS_IV]]
; CHECK-NEXT: [[IN1:%.*]] = load double, ptr [[ARRAYIDX]], align 8
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[INPUT2]], i64 0, i64 [[INDVARS_IV]]
@@ -471,19 +449,15 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
;
; ZVFHMIN-LABEL: define void @fmax64(
; ZVFHMIN-SAME: ptr noundef readonly captures(none) [[INPUT1:%.*]], ptr noundef readonly captures(none) [[INPUT2:%.*]], ptr noundef writeonly captures(none) [[OUTPUT:%.*]]) #[[ATTR0]] {
-; ZVFHMIN-NEXT: [[ENTRY:.*]]:
+; ZVFHMIN-NEXT: [[ENTRY:.*:]]
; ZVFHMIN-NEXT: [[INPUT23:%.*]] = ptrtoaddr ptr [[INPUT2]] to i64
; ZVFHMIN-NEXT: [[INPUT12:%.*]] = ptrtoaddr ptr [[INPUT1]] to i64
; ZVFHMIN-NEXT: [[OUTPUT1:%.*]] = ptrtoaddr ptr [[OUTPUT]] to i64
-; ZVFHMIN-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; ZVFHMIN-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 17)
-; ZVFHMIN-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
-; ZVFHMIN-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; ZVFHMIN-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
-; ZVFHMIN-NEXT: [[TMP14:%.*]] = add i64 [[TMP4]], -1
+; ZVFHMIN-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP14:%.*]] = add nsw i64 [[TMP2]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP12:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP12]], [[TMP14]]
@@ -491,8 +465,10 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: [[TMP16:%.*]] = sub i64 [[TMP8]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP16]], [[TMP14]]
; ZVFHMIN-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; ZVFHMIN-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; ZVFHMIN: [[VECTOR_PH]]:
+; ZVFHMIN-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 1
; ZVFHMIN-NEXT: br label %[[VECTOR_BODY:.*]]
; ZVFHMIN: [[VECTOR_BODY]]:
; ZVFHMIN-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -509,10 +485,9 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN: [[MIDDLE_BLOCK]]:
; ZVFHMIN-NEXT: br label %[[EXIT:.*]]
; ZVFHMIN: [[SCALAR_PH]]:
-; ZVFHMIN-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; ZVFHMIN-NEXT: br label %[[FOR_BODY:.*]]
; ZVFHMIN: [[FOR_BODY]]:
-; ZVFHMIN-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; ZVFHMIN-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
; ZVFHMIN-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[INPUT1]], i64 0, i64 [[IV]]
; ZVFHMIN-NEXT: [[IN1:%.*]] = load double, ptr [[ARRAYIDX]], align 8
; ZVFHMIN-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[INPUT2]], i64 0, i64 [[IV]]
@@ -547,7 +522,7 @@ exit:
}
-define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) {
+define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) vscale_range(2, 1024) {
; CHECK-LABEL: define void @fmin16(
; CHECK-SAME: ptr noundef readonly captures(none) [[INPUT1:%.*]], ptr noundef readonly captures(none) [[INPUT2:%.*]], ptr noundef writeonly captures(none) [[OUTPUT:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -557,8 +532,8 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP15]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP3]], -1
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP3]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
@@ -613,8 +588,8 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl i64 [[TMP3]], 4
-; ZVFHMIN-NEXT: [[TMP11:%.*]] = add i64 [[TMP1]], -1
+; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP11:%.*]] = add nsw i64 [[TMP1]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
@@ -681,7 +656,7 @@ exit:
}
-define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) {
+define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef readonly captures(none) %input2, ptr noundef writeonly captures(none) %output) vscale_range(2, 1024) {
; CHECK-LABEL: define void @fmax16(
; CHECK-SAME: ptr noundef readonly captures(none) [[INPUT1:%.*]], ptr noundef readonly captures(none) [[INPUT2:%.*]], ptr noundef writeonly captures(none) [[OUTPUT:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -691,8 +666,8 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP15]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP3]], -1
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP3]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
@@ -747,8 +722,8 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
; ZVFHMIN-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; ZVFHMIN: [[VECTOR_MEMCHECK]]:
; ZVFHMIN-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl i64 [[TMP3]], 4
-; ZVFHMIN-NEXT: [[TMP11:%.*]] = add i64 [[TMP1]], -1
+; ZVFHMIN-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; ZVFHMIN-NEXT: [[TMP11:%.*]] = add nsw i64 [[TMP1]], -1
; ZVFHMIN-NEXT: [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
; ZVFHMIN-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; ZVFHMIN-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP11]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
index c1ed5b1327073..fda9e244306a1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
@@ -105,7 +105,7 @@ define void @predicated_strided_store(ptr %start) {
; RVA23-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[TMP1]]
; RVA23-NEXT: call void @llvm.experimental.vp.strided.store.nxv16i8.p0.i64(<vscale x 16 x i8> zeroinitializer, ptr align 1 [[TMP2]], i64 7, <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
; RVA23-NEXT: [[TMP3:%.*]] = zext i32 [[TMP0]] to i64
-; RVA23-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP3]], [[INDEX]]
+; RVA23-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP3]], [[INDEX]]
; RVA23-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
; RVA23-NEXT: [[TMP4:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; RVA23-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
@@ -127,7 +127,7 @@ define void @predicated_strided_store(ptr %start) {
; RVA23ZVL1024B-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[TMP1]]
; RVA23ZVL1024B-NEXT: call void @llvm.experimental.vp.strided.store.nxv8i8.p0.i64(<vscale x 8 x i8> zeroinitializer, ptr align 1 [[TMP2]], i64 7, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
; RVA23ZVL1024B-NEXT: [[TMP3:%.*]] = zext i32 [[TMP0]] to i64
-; RVA23ZVL1024B-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP3]], [[INDEX]]
+; RVA23ZVL1024B-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP3]], [[INDEX]]
; RVA23ZVL1024B-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
; RVA23ZVL1024B-NEXT: [[TMP4:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; RVA23ZVL1024B-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
@@ -279,6 +279,3 @@ exit:
}
attributes #0 = { "target-features"="+zve64x,+zvl256b" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; RVA23: {{.*}}
-; RVA23ZVL1024B: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
index 3a796fe80af11..aa5734371f0d5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
@@ -233,8 +233,8 @@ exit:
-attributes #0 = { "target-features"="+64bit,+v,+zvl256b" }
-attributes #1 = { "target-cpu"="sifive-p670" }
+attributes #0 = { "target-features"="+64bit,+v,+zvl256b" vscale_range(4, 1024) }
+attributes #1 = { "target-cpu"="sifive-p670" vscale_range(2, 1024) }
;.
; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/indvar-overflow-check-scalable-tc.ll b/llvm/test/Transforms/LoopVectorize/RISCV/indvar-overflow-check-scalable-tc.ll
index 6ec22b0696fa9..15345520b0fe2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/indvar-overflow-check-scalable-tc.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/indvar-overflow-check-scalable-tc.ll
@@ -2,7 +2,7 @@
; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s
; VF is scalable, as is TC. We can prove that TC > VF.
-define void @iv_no_overflow(ptr noalias %a, ptr noalias %b) #0 {
+define void @iv_no_overflow(ptr noalias %a, ptr noalias %b) vscale_range(2, 1024) {
; CHECK-LABEL: define void @iv_no_overflow(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -49,7 +49,7 @@ exit:
}
; VF is scalable, as is TC. TC * MaxVScale overflows in i32, but not the i64 induction.
-define void @tc_maxvscale_overflow(ptr noalias %a, ptr noalias %b) #0 {
+define void @tc_maxvscale_overflow(ptr noalias %a, ptr noalias %b) vscale_range(2, 1024) {
; CHECK-LABEL: define void @tc_maxvscale_overflow(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -97,7 +97,7 @@ exit:
; VF is scalable, as is TC. TC * MaxVScale (8388609 * 1024) is not
; representable in the type of the i32 induction variable.
-define void @tc_maxvscale_exceeds_iv_type(ptr noalias %a, ptr noalias %b) #0 {
+define void @tc_maxvscale_exceeds_iv_type(ptr noalias %a, ptr noalias %b) vscale_range(2, 1024) {
; CHECK-LABEL: define void @tc_maxvscale_exceeds_iv_type(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
index 6b22001dde07b..e8f02f97cabc1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
@@ -8,7 +8,7 @@
target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
target triple = "riscv64"
-define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
+define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) vscale_range(2, 1024) {
; OUTLOOP-LABEL: @add_i16_i32(
; OUTLOOP-NEXT: entry:
; OUTLOOP-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
@@ -16,8 +16,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
; OUTLOOP: for.body.preheader:
; OUTLOOP-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
; OUTLOOP-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; OUTLOOP-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 8)
-; OUTLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], [[UMAX]]
+; OUTLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], [[TMP1]]
; OUTLOOP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; OUTLOOP: vector.ph:
; OUTLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP1]]
@@ -65,8 +64,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
; INLOOP: for.body.preheader:
; INLOOP-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
; INLOOP-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 3
-; INLOOP-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 16)
-; INLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], [[UMAX]]
+; INLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], [[TMP1]]
; INLOOP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; INLOOP: vector.ph:
; INLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP1]]
@@ -188,13 +186,12 @@ for.cond.cleanup:
ret i32 %r.0.lcssa
}
-define i32 @smin(ptr %a, i64 %n, i32 %start) {
+define i32 @smin(ptr %a, i64 %n, i32 %start) vscale_range(2, 1024) {
; OUTLOOP-LABEL: @smin(
; OUTLOOP-NEXT: entry:
; OUTLOOP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; OUTLOOP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; OUTLOOP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; OUTLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[UMAX]]
+; OUTLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
; OUTLOOP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; OUTLOOP: vector.ph:
; OUTLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
@@ -236,8 +233,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
; INLOOP-NEXT: entry:
; INLOOP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; INLOOP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; INLOOP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; INLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[UMAX]]
+; INLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
; INLOOP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; INLOOP: vector.ph:
; INLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index 65cc3b25c3d04..b36bf1ab2320c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -3,7 +3,7 @@
; RUN: opt < %s -passes=loop-vectorize -scalable-vectorization=off -mtriple=riscv64 -mattr=+v -S | FileCheck %s --check-prefix=FIXED
; RUN: opt < %s -passes=loop-vectorize -scalable-vectorization=on -mtriple=riscv64 -mattr=+v -S | FileCheck %s --check-prefix=SCALABLE
-define void @load_store_factor2_i32(ptr %p) {
+define void @load_store_factor2_i32(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @load_store_factor2_i32(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -115,7 +115,7 @@ exit:
ret void
}
-define void @load_store_factor2_i64(ptr %p) {
+define void @load_store_factor2_i64(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @load_store_factor2_i64(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -227,7 +227,7 @@ exit:
ret void
}
-define void @load_store_factor3_i32(ptr %p) {
+define void @load_store_factor3_i32(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @load_store_factor3_i32(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -353,7 +353,7 @@ exit:
ret void
}
-define void @load_store_factor3_i64(ptr %p) {
+define void @load_store_factor3_i64(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @load_store_factor3_i64(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -479,7 +479,7 @@ exit:
ret void
}
-define void @load_store_factor4(ptr %p) {
+define void @load_store_factor4(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @load_store_factor4(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -606,7 +606,7 @@ exit:
ret void
}
-define void @load_store_factor5(ptr %p) {
+define void @load_store_factor5(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @load_store_factor5(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -777,7 +777,7 @@ exit:
ret void
}
-define void @load_store_factor6(ptr %p) {
+define void @load_store_factor6(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @load_store_factor6(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -969,7 +969,7 @@ exit:
ret void
}
-define void @load_store_factor7(ptr %p) {
+define void @load_store_factor7(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @load_store_factor7(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -1182,7 +1182,7 @@ exit:
ret void
}
-define void @load_store_factor8(ptr %p) {
+define void @load_store_factor8(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @load_store_factor8(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -1416,7 +1416,7 @@ exit:
ret void
}
-define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
+define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) vscale_range(2, 1024) {
; CHECK-LABEL: @combine_load_factor2_i32(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -1523,7 +1523,7 @@ exit:
ret void
}
-define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
+define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) vscale_range(2, 1024) {
; CHECK-LABEL: @combine_load_factor2_i64(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
index 804fe357f5970..aca1e5e1014f2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
@@ -5,15 +5,14 @@
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
-define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) {
+define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) vscale_range(2, 1024) {
; SCALAR_EPILOGUE-LABEL: define void @masked_strided_factor2(
; SCALAR_EPILOGUE-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] {
; SCALAR_EPILOGUE-NEXT: [[ENTRY:.*:]]
; SCALAR_EPILOGUE-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; SCALAR_EPILOGUE-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
; SCALAR_EPILOGUE-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
-; SCALAR_EPILOGUE-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 32)
-; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP2]]
+; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP1]]
; SCALAR_EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; SCALAR_EPILOGUE: [[VECTOR_PH]]:
; SCALAR_EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP1]]
@@ -177,15 +176,14 @@ for.end:
}
-define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) {
+define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) vscale_range(2, 1024) {
; SCALAR_EPILOGUE-LABEL: define void @masked_strided_factor4(
; SCALAR_EPILOGUE-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
; SCALAR_EPILOGUE-NEXT: [[ENTRY:.*:]]
; SCALAR_EPILOGUE-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; SCALAR_EPILOGUE-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
; SCALAR_EPILOGUE-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
-; SCALAR_EPILOGUE-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 32)
-; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP2]]
+; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP1]]
; SCALAR_EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; SCALAR_EPILOGUE: [[VECTOR_PH]]:
; SCALAR_EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll b/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll
index 755dc7ccd77d2..9479f934eb3f9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll
@@ -8,33 +8,24 @@
define void @load_store(ptr %p) {
; LMUL1-LABEL: @load_store(
; LMUL1-NEXT: entry:
-; LMUL1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
-; LMUL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; LMUL1-NEXT: br label [[VECTOR_PH:%.*]]
; LMUL1: vector.ph:
; LMUL1-NEXT: br label [[VECTOR_BODY:%.*]]
; LMUL1: vector.body:
; LMUL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; LMUL1-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; LMUL1-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
; LMUL1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 [[INDEX]]
-; LMUL1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 1 x i64>, ptr [[TMP3]], align 8
+; LMUL1-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 1 x i64> @llvm.vp.load.nxv1i64.p0(ptr align 8 [[TMP3]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP0]])
; LMUL1-NEXT: [[TMP5:%.*]] = add <vscale x 1 x i64> [[WIDE_LOAD]], splat (i64 1)
-; LMUL1-NEXT: store <vscale x 1 x i64> [[TMP5]], ptr [[TMP3]], align 8
-; LMUL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
-; LMUL1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; LMUL1-NEXT: call void @llvm.vp.store.nxv1i64.p0(<vscale x 1 x i64> [[TMP5]], ptr align 8 [[TMP3]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP0]])
+; LMUL1-NEXT: [[TMP6:%.*]] = zext i32 [[TMP0]] to i64
+; LMUL1-NEXT: [[INDEX_NEXT]] = add i64 [[TMP6]], [[INDEX]]
+; LMUL1-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP6]]
+; LMUL1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; LMUL1-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; LMUL1: middle.block:
-; LMUL1-NEXT: br label [[FOR_END:%.*]]
-; LMUL1: scalar.ph:
; LMUL1-NEXT: br label [[FOR_BODY:%.*]]
-; LMUL1: for.body:
-; LMUL1-NEXT: [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; LMUL1-NEXT: [[Q:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[IV]]
-; LMUL1-NEXT: [[V:%.*]] = load i64, ptr [[Q]], align 8
-; LMUL1-NEXT: [[W:%.*]] = add i64 [[V]], 1
-; LMUL1-NEXT: store i64 [[W]], ptr [[Q]], align 8
-; LMUL1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; LMUL1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; LMUL1-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; LMUL1: for.end:
; LMUL1-NEXT: ret void
;
@@ -52,7 +43,7 @@ define void @load_store(ptr %p) {
; LMUL2-NEXT: [[TMP7:%.*]] = add <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 1)
; LMUL2-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP7]], ptr align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP6]])
; LMUL2-NEXT: [[TMP10:%.*]] = zext i32 [[TMP6]] to i64
-; LMUL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
+; LMUL2-NEXT: [[INDEX_NEXT]] = add i64 [[TMP10]], [[INDEX]]
; LMUL2-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; LMUL2-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; LMUL2-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -75,7 +66,7 @@ define void @load_store(ptr %p) {
; LMUL4-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i64> [[WIDE_LOAD]], splat (i64 1)
; LMUL4-NEXT: call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[TMP7]], ptr align 8 [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
; LMUL4-NEXT: [[TMP10:%.*]] = zext i32 [[TMP6]] to i64
-; LMUL4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
+; LMUL4-NEXT: [[INDEX_NEXT]] = add i64 [[TMP10]], [[INDEX]]
; LMUL4-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; LMUL4-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; LMUL4-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -98,7 +89,7 @@ define void @load_store(ptr %p) {
; LMUL8-NEXT: [[TMP7:%.*]] = add <vscale x 8 x i64> [[WIDE_LOAD]], splat (i64 1)
; LMUL8-NEXT: call void @llvm.vp.store.nxv8i64.p0(<vscale x 8 x i64> [[TMP7]], ptr align 8 [[TMP5]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
; LMUL8-NEXT: [[TMP10:%.*]] = zext i32 [[TMP6]] to i64
-; LMUL8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
+; LMUL8-NEXT: [[INDEX_NEXT]] = add i64 [[TMP10]], [[INDEX]]
; LMUL8-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; LMUL8-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; LMUL8-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll b/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll
index 609e8143c9d52..26b9f2038b688 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll
@@ -8,7 +8,7 @@ target triple = "riscv64"
; the data width. This is correct, but sub-optimal as it causes a vsetvli
; toggle in the generated code for no reason. We could have used a i32
; element type for the index here and matched the data width.
-define void @test(ptr noalias nocapture %a, ptr noalias nocapture %b, i32 %v) {
+define void @test(ptr noalias nocapture %a, ptr noalias nocapture %b, i32 %v) vscale_range(2, 1024) {
; VLENUNK-LABEL: @test(
; VLENUNK-NEXT: entry:
; VLENUNK-NEXT: br label [[VECTOR_PH:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
index 5ef175eaca194..7b6b26f300c94 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
@@ -13,7 +13,7 @@
; }
;}
-define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture readonly %trigger) #0 {
+define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture readonly %trigger) vscale_range(4, 1024) {
; RV32-LABEL: @foo4(
; RV32-NEXT: entry:
; RV32-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
index e99c68eeb66cd..a8546b83e6e77 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
@@ -9,14 +9,13 @@
target triple = "riscv64-none-unknown-elf"
-define i32 @vdot4a(ptr %a, ptr %b) #0 {
+define i32 @vdot4a(ptr %a, ptr %b) vscale_range(2, 1024) {
; V-LABEL: define i32 @vdot4a(
; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; V-NEXT: entry:
; V-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; V-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; V-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; V-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; V: vector.ph:
; V-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -47,8 +46,7 @@ define i32 @vdot4a(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: entry:
; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; ZVDOT4A8I-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; ZVDOT4A8I-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; ZVDOT4A8I-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; ZVDOT4A8I: vector.ph:
; ZVDOT4A8I-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -200,14 +198,13 @@ for.exit:
}
-define i32 @vdot4au(ptr %a, ptr %b) #0 {
+define i32 @vdot4au(ptr %a, ptr %b) vscale_range(2, 1024) {
; V-LABEL: define i32 @vdot4au(
; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
; V-NEXT: entry:
; V-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; V-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; V-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; V-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; V: vector.ph:
; V-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -238,8 +235,7 @@ define i32 @vdot4au(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: entry:
; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; ZVDOT4A8I-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; ZVDOT4A8I-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; ZVDOT4A8I-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; ZVDOT4A8I: vector.ph:
; ZVDOT4A8I-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -391,14 +387,13 @@ for.exit:
}
-define i32 @vdot4asu(ptr %a, ptr %b) #0 {
+define i32 @vdot4asu(ptr %a, ptr %b) vscale_range(2, 1024) {
; V-LABEL: define i32 @vdot4asu(
; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
; V-NEXT: entry:
; V-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; V-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; V-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; V-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; V: vector.ph:
; V-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -429,8 +424,7 @@ define i32 @vdot4asu(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: entry:
; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; ZVDOT4A8I-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; ZVDOT4A8I-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; ZVDOT4A8I-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; ZVDOT4A8I: vector.ph:
; ZVDOT4A8I-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -581,14 +575,13 @@ for.exit:
ret i32 %add
}
-define i32 @vdot4asu2(ptr %a, ptr %b) #0 {
+define i32 @vdot4asu2(ptr %a, ptr %b) vscale_range(2, 1024) {
; V-LABEL: define i32 @vdot4asu2(
; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
; V-NEXT: entry:
; V-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; V-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; V-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; V-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; V-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; V: vector.ph:
; V-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -619,8 +612,7 @@ define i32 @vdot4asu2(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: entry:
; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; ZVDOT4A8I-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; ZVDOT4A8I-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; ZVDOT4A8I-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; ZVDOT4A8I-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; ZVDOT4A8I: vector.ph:
; ZVDOT4A8I-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
index 19a2424a9ee7f..326ac9afbb9b6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
@@ -5,7 +5,7 @@ target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
target triple = "riscv64-unknown-linux-gnu"
; Test case for https://github.com/llvm/llvm-project/issues/87378.
-define void @pr87378_vpinstruction_or_drop_poison_generating_flags(ptr %arg, i64 %a, i64 %b, i64 %c) {
+define void @pr87378_vpinstruction_or_drop_poison_generating_flags(ptr %arg, i64 %a, i64 %b, i64 %c) vscale_range(2, 1024) {
; CHECK-LABEL: define void @pr87378_vpinstruction_or_drop_poison_generating_flags(
; CHECK-SAME: ptr [[ARG:%.*]], i64 [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
index 2316a478becc5..72d09364397a5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
@@ -5,7 +5,7 @@
; BranchProbabilityInfo, but if we vectorize it then we will unconditionally
; execute it. Avoid this unprofitable vectorization by taking the nested
; probability into account in the cost model.
-define void @nested(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) {
+define void @nested(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) vscale_range(2, 1024) {
; CHECK-LABEL: define void @nested(
; CHECK-SAME: ptr noalias [[P0:%.*]], ptr noalias [[P1:%.*]], i1 [[C0:%.*]], i1 [[C1:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -63,7 +63,7 @@ exit:
; This is the same CFG as @nested above, but we have provided branch weights
; which tell BranchProbabilityInfo that then.1 will always be taken. In this
; case, we should vectorize because it is profitable.
-define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) {
+define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) vscale_range(2, 1024) {
; CHECK-LABEL: define void @always_taken(
; CHECK-SAME: ptr noalias [[P0:%.*]], ptr noalias [[P1:%.*]], i1 [[C0:%.*]], i1 [[C1:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll
index c3da8dcff27ec..aaa2fa0659024 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-prune-vf.ll
@@ -3,7 +3,7 @@
; RUN: opt -p loop-vectorize -mtriple riscv64 -mattr=+v -vectorizer-consider-reg-pressure=true -S < %s | FileCheck %s
; RUN: opt -p loop-vectorize -mtriple riscv64 -mattr=+v -vectorizer-consider-reg-pressure=false -S < %s | FileCheck %s --check-prefix=NO-REG-PRESSURE-CHECK
-define void @f(ptr noalias %p0, ptr noalias %p1, ptr noalias %p2) {
+define void @f(ptr noalias %p0, ptr noalias %p1, ptr noalias %p2) vscale_range(2, 1024) {
; CHECK-LABEL: define void @f(
; CHECK-SAME: ptr noalias [[P0:%.*]], ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index a9206afcacdd7..f4b68fcec1261 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -15,7 +15,7 @@
; RUN: -force-vector-interleave=2 -S < %s \
; RUN: | FileCheck --check-prefix=RV64-UF2 %s
-define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
+define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) vscale_range(2, 1024) {
; RV64-LABEL: define void @vector_reverse_i32(
; RV64-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; RV64-NEXT: [[ENTRY:.*:]]
@@ -139,7 +139,7 @@ exit:
ret void
}
-define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) {
+define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) vscale_range(2, 1024) {
; RV64-LABEL: define void @vector_reverse_i64(
; RV64-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i32 noundef signext [[N:%.*]]) #[[ATTR0]] {
; RV64-NEXT: [[ENTRY:.*:]]
@@ -161,8 +161,8 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV64-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; RV64: [[VECTOR_MEMCHECK]]:
; RV64-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT: [[TMP10:%.*]] = shl i64 [[TMP11]], 4
-; RV64-NEXT: [[TMP16:%.*]] = add i64 [[TMP10]], -1
+; RV64-NEXT: [[TMP10:%.*]] = shl nuw nsw i64 [[TMP11]], 4
+; RV64-NEXT: [[TMP16:%.*]] = add nsw i64 [[TMP10]], -1
; RV64-NEXT: [[TMP14:%.*]] = sub i64 [[B1]], [[A2]]
; RV64-NEXT: [[TMP15:%.*]] = sub i64 [[TMP14]], 1
; RV64-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP15]], [[TMP16]]
@@ -213,8 +213,8 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV32-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; RV32: [[VECTOR_MEMCHECK]]:
; RV32-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; RV32-NEXT: [[TMP2:%.*]] = shl i32 [[TMP3]], 4
-; RV32-NEXT: [[TMP8:%.*]] = add i32 [[TMP2]], -1
+; RV32-NEXT: [[TMP2:%.*]] = shl nuw nsw i32 [[TMP3]], 4
+; RV32-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP2]], -1
; RV32-NEXT: [[TMP6:%.*]] = sub i32 [[B1]], [[A2]]
; RV32-NEXT: [[TMP7:%.*]] = sub i32 [[TMP6]], 1
; RV32-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i32 [[TMP7]], [[TMP8]]
@@ -277,8 +277,8 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV64-UF2-NEXT: br i1 [[TMP10]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; RV64-UF2: [[VECTOR_MEMCHECK]]:
; RV64-UF2-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT: [[TMP12:%.*]] = shl i64 [[TMP11]], 5
-; RV64-UF2-NEXT: [[TMP15:%.*]] = add i64 [[TMP12]], -1
+; RV64-UF2-NEXT: [[TMP12:%.*]] = shl nuw nsw i64 [[TMP11]], 5
+; RV64-UF2-NEXT: [[TMP15:%.*]] = add nsw i64 [[TMP12]], -1
; RV64-UF2-NEXT: [[TMP14:%.*]] = sub i64 [[B1]], [[A2]]
; RV64-UF2-NEXT: [[TMP16:%.*]] = sub i64 [[TMP14]], 1
; RV64-UF2-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP16]], [[TMP15]]
@@ -354,7 +354,7 @@ for.body:
br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
}
-define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) {
+define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) vscale_range(2, 1024) {
; RV64-LABEL: define void @vector_reverse_f32(
; RV64-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i32 noundef signext [[N:%.*]]) #[[ATTR0]] {
; RV64-NEXT: [[ENTRY:.*:]]
@@ -376,8 +376,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV64-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; RV64: [[VECTOR_MEMCHECK]]:
; RV64-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT: [[TMP10:%.*]] = shl i64 [[TMP11]], 4
-; RV64-NEXT: [[TMP16:%.*]] = add i64 [[TMP10]], -1
+; RV64-NEXT: [[TMP10:%.*]] = shl nuw nsw i64 [[TMP11]], 4
+; RV64-NEXT: [[TMP16:%.*]] = add nsw i64 [[TMP10]], -1
; RV64-NEXT: [[TMP14:%.*]] = sub i64 [[B1]], [[A2]]
; RV64-NEXT: [[TMP15:%.*]] = sub i64 [[TMP14]], 1
; RV64-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP15]], [[TMP16]]
@@ -428,8 +428,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV32-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; RV32: [[VECTOR_MEMCHECK]]:
; RV32-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; RV32-NEXT: [[TMP2:%.*]] = shl i32 [[TMP3]], 4
-; RV32-NEXT: [[TMP8:%.*]] = add i32 [[TMP2]], -1
+; RV32-NEXT: [[TMP2:%.*]] = shl nuw nsw i32 [[TMP3]], 4
+; RV32-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP2]], -1
; RV32-NEXT: [[TMP6:%.*]] = sub i32 [[B1]], [[A2]]
; RV32-NEXT: [[TMP7:%.*]] = sub i32 [[TMP6]], 1
; RV32-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i32 [[TMP7]], [[TMP8]]
@@ -492,8 +492,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; RV64-UF2-NEXT: br i1 [[TMP10]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; RV64-UF2: [[VECTOR_MEMCHECK]]:
; RV64-UF2-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT: [[TMP12:%.*]] = shl i64 [[TMP11]], 5
-; RV64-UF2-NEXT: [[TMP15:%.*]] = add i64 [[TMP12]], -1
+; RV64-UF2-NEXT: [[TMP12:%.*]] = shl nuw nsw i64 [[TMP11]], 5
+; RV64-UF2-NEXT: [[TMP15:%.*]] = add nsw i64 [[TMP12]], -1
; RV64-UF2-NEXT: [[TMP14:%.*]] = sub i64 [[B1]], [[A2]]
; RV64-UF2-NEXT: [[TMP16:%.*]] = sub i64 [[TMP14]], 1
; RV64-UF2-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP16]], [[TMP15]]
@@ -569,7 +569,7 @@ for.body:
br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
}
-define void @vector_reverse_f32_simplify(ptr noalias %A, ptr noalias %B) {
+define void @vector_reverse_f32_simplify(ptr noalias %A, ptr noalias %B) vscale_range(2, 1024) {
; RV64-LABEL: define void @vector_reverse_f32_simplify(
; RV64-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
; RV64-NEXT: [[ENTRY:.*:]]
@@ -693,7 +693,7 @@ exit:
ret void
}
-define void @vector_reverse_irregular_type(ptr noalias %A, ptr noalias %B) {
+define void @vector_reverse_irregular_type(ptr noalias %A, ptr noalias %B) vscale_range(2, 1024) {
; RV64-LABEL: define void @vector_reverse_irregular_type(
; RV64-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
; RV64-NEXT: [[ENTRY:.*:]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
index a31bb408dcbc7..a1585db9c160c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
@@ -4,7 +4,7 @@
; TODO: Make sure that optimizations (unit-stride multiversioning) don't prohibit vectorization.
-define void @foo(ptr %p, ptr %p.strided, i64 %n, i64 %stride) {
+define void @foo(ptr %p, ptr %p.strided, i64 %n, i64 %stride) vscale_range(2, 1024) {
; UNIT-STRIDE-MV-LABEL: define void @foo(
; UNIT-STRIDE-MV-SAME: ptr [[P:%.*]], ptr [[P_STRIDED:%.*]], i64 [[N:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
; UNIT-STRIDE-MV-NEXT: [[ENTRY:.*]]:
@@ -87,7 +87,7 @@ define void @foo(ptr %p, ptr %p.strided, i64 %n, i64 %stride) {
; NO-UNIT-STRIDE-MV-NEXT: [[INDUCTION:%.*]] = add nsw <vscale x 2 x i64> splat (i64 1), [[TMP35]]
; NO-UNIT-STRIDE-MV-NEXT: br label %[[HEADER:.*]]
; NO-UNIT-STRIDE-MV: [[HEADER]]:
-; NO-UNIT-STRIDE-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[HEADER]] ]
+; NO-UNIT-STRIDE-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[STEP_ADD_3:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[AVL:%.*]] = phi i64 [ 63, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP25:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
@@ -103,7 +103,7 @@ define void @foo(ptr %p, ptr %p.strided, i64 %n, i64 %stride) {
; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_MASKED_GATHER22:%.*]] = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP40]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META3:![0-9]+]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP49:%.*]] = add <vscale x 2 x i64> [[WIDE_LOAD19]], [[WIDE_MASKED_GATHER22]]
; NO-UNIT-STRIDE-MV-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP49]], ptr align 8 [[TMP31]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
-; NO-UNIT-STRIDE-MV-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP26]], [[INDEX]]
+; NO-UNIT-STRIDE-MV-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP26]], [[INDEX]]
; NO-UNIT-STRIDE-MV-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
; NO-UNIT-STRIDE-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll b/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll
index 6a0318620f95a..df789d38e4b36 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll
@@ -7,7 +7,7 @@ target triple = "riscv64"
; Dependence distance between read and write is greater than the trip
; count of the loop. Thus, values written are never read for any
; valid vectorization of the loop.
-define void @test(ptr %p) {
+define void @test(ptr %p) vscale_range(4, 1024) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -52,7 +52,7 @@ exit:
; Dependence distance is less than trip count, thus we must prove that
; chosen VF guaranteed to be less than dependence distance.
-define void @test_may_clobber(ptr %p) {
+define void @test_may_clobber(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @test_may_clobber(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -92,7 +92,7 @@ exit:
}
; Trviailly no overlap due to maximum possible value of VLEN and LMUL
-define void @trivial_due_max_vscale(ptr %p) {
+define void @trivial_due_max_vscale(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @trivial_due_max_vscale(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -136,7 +136,7 @@ exit:
}
; Dependence distance could be violated via LMUL>=2 or interleaving
-define void @no_high_lmul_or_interleave(ptr %p) {
+define void @no_high_lmul_or_interleave(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: @no_high_lmul_or_interleave(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -182,7 +182,7 @@ exit:
@a = external global [10 x [12 x i16]]
; Test case for https://github.com/llvm/llvm-project/issues/134696.
-define void @safe_load_store_distance_not_pow_of_2(i64 %N) {
+define void @safe_load_store_distance_not_pow_of_2(i64 %N) vscale_range(2, 1024) {
; CHECK-LABEL: @safe_load_store_distance_not_pow_of_2(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N:%.*]], i64 1)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll
index 7330ce61515d9..3b7d255106ecb 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll
@@ -5,7 +5,7 @@
; allowed. The primary goal of this is check for crashes during cost modeling, but it also
; exercises the default heuristics in a useful way.
-define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) {
+define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @vector_add(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -51,7 +51,7 @@ for.end:
; Same as above, but with op type of i32. We currently have a bug around
; etype=ELEN profitability in the vectorizer, and having a smaller element
; width test allows us to highlight different aspects of codegen.
-define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) {
+define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @vector_add_i32(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -71,7 +71,7 @@ define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.end:
@@ -97,7 +97,7 @@ for.end:
; a[b[i]] += v, mostly to exercise scatter/gather costing
; TODO: Currently fails to vectorize due to a memory conflict
-define void @indexed_add(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
+define void @indexed_add(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @indexed_add(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
@@ -135,7 +135,7 @@ for.end:
}
; a[b[i]] = v, exercise scatter support
-define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
+define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @indexed_store(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -155,7 +155,7 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.end:
@@ -178,7 +178,7 @@ for.end:
ret void
}
-define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
+define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @indexed_load(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -199,7 +199,7 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP9]])
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
@@ -225,7 +225,7 @@ for.end:
ret i64 %sum.next
}
-define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) {
+define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @splat_int(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -243,7 +243,7 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.end:
@@ -264,7 +264,7 @@ for.end:
ret void
}
-define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) {
+define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @splat_ptr(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -282,7 +282,7 @@ define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.end:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
index 8249d24bd9d89..99b0cac0b59bc 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
@@ -6,7 +6,7 @@
target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
target triple = "riscv64"
-define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) {
+define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @vector_add(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -51,7 +51,7 @@ for.end:
; a[b[i]] = v, exercise scatter support
-define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
+define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @indexed_store(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -94,7 +94,7 @@ for.end:
ret void
}
-define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
+define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @indexed_load(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -141,7 +141,7 @@ for.end:
ret i64 %sum.next
}
-define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) {
+define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @splat_int(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -180,7 +180,7 @@ for.end:
ret void
}
-define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
+define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @uniform_store(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
@@ -221,7 +221,7 @@ for.end:
ret void
}
-define i64 @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) {
+define i64 @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @uniform_load(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[ENTRY:%.*]]
@@ -263,7 +263,7 @@ for.end:
}
-define void @vector_add_trip1024(ptr noalias nocapture %a, i64 %v, i64 %n) {
+define void @vector_add_trip1024(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: @vector_add_trip1024(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-vf-hint.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-vf-hint.ll
index 8fbfed9b9c636..3373dd45ddb3d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-vf-hint.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-vf-hint.ll
@@ -1,6 +1,4 @@
-; RUN: opt -mtriple=riscv64 -mattr=+m,+v -passes=loop-vectorize \
-; RUN: -riscv-v-vector-bits-max=512 -S -scalable-vectorization=on < %s 2>&1 \
-; RUN: | FileCheck %s
+; RUN: opt -mtriple=riscv64 -mattr=+v -passes=loop-vectorize -S %s | FileCheck %s
; void test(int *a, int *b, int N) {
; #pragma clang loop vectorize(enable) vectorize_width(2, scalable)
@@ -10,26 +8,25 @@
; }
;
; CHECK: <vscale x 2 x i32>
-define void @test(ptr %a, ptr %b) {
+define i32 @test(ptr %p, i32 %n) vscale_range(2, 1024) {
entry:
- br label %loop
+ %skip = icmp eq i32 %n, 0
+ br i1 %skip, label %exit, label %loop
loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %arrayidx2 = getelementptr inbounds i32, ptr %b, i64 %iv
- %1 = load i32, ptr %arrayidx2, align 4
- %add = add nsw i32 %1, %0
- %2 = add nuw nsw i64 %iv, 64
- %arrayidx5 = getelementptr inbounds i32, ptr %a, i64 %2
- store i32 %add, ptr %arrayidx5, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond.not = icmp eq i64 %iv.next, 1024
- br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !6
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i32 [ 0, %entry ], [ %rdx.next, %loop ]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep
+ %cmp = icmp eq i32 %x, 0
+ %rdx.next = select i1 %cmp, i32 1, i32 %rdx
+ %iv.next = add nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop, !llvm.loop !6
exit:
- ret void
+ %res = phi i32 [ 0, %entry ], [ %rdx.next, %loop ]
+ ret i32 %res
}
!6 = !{!6, !7, !8}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
index d93f1ce0bdfa7..0b9617479b0de 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
@@ -6,7 +6,7 @@
; constant, but the pointer stride exceeds INT32_MAX on rv64 with a narrow IV.
; TODO: Should form a strided load with SCEV predicates.
-define void @stride_sext_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst, i16 %start) {
+define void @stride_sext_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst, i16 %start) vscale_range(4, 1024) {
; CHECK-LABEL: define void @stride_sext_exceeds_i32_max(
; CHECK-SAME: ptr noalias readonly [[SRC:%.*]], ptr noalias [[DST:%.*]], i16 [[START:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -61,7 +61,7 @@ exit:
ret void
}
-define void @stride_zext_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst, i16 %start) {
+define void @stride_zext_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst, i16 %start) vscale_range(4, 1024) {
; CHECK-LABEL: define void @stride_zext_exceeds_i32_max(
; CHECK-SAME: ptr noalias readonly [[SRC:%.*]], ptr noalias [[DST:%.*]], i16 [[START:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll
index 183620729a254..c1fb7eb61ef39 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-i64-rv32.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize -mtriple=riscv32 -mattr=+v < %s -S | FileCheck %s
-define void @single_constant_stride_int_scaled(ptr %p) {
+define void @single_constant_stride_int_scaled(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: define void @single_constant_stride_int_scaled(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll
index 708f488b93eef..9cfed35979ee2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll
@@ -2,7 +2,7 @@
; RUN: opt < %s -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -S | FileCheck --check-prefix=RV64 %s
; RUN: opt < %s -passes=loop-vectorize -mtriple=riscv32 -mattr=+v -S | FileCheck --check-prefix=RV32 %s
-define void @narrow_iv_i8_sext_i64(ptr noalias %arr, ptr noalias %out) {
+define void @narrow_iv_i8_sext_i64(ptr noalias %arr, ptr noalias %out) vscale_range(2, 1024) {
; RV64-LABEL: define void @narrow_iv_i8_sext_i64(
; RV64-SAME: ptr noalias [[ARR:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
; RV64-NEXT: [[ENTRY:.*:]]
@@ -12,18 +12,9 @@ define void @narrow_iv_i8_sext_i64(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
-; RV64-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV64-NEXT: [[AVL:%.*]] = phi i32 [ 16, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
-; RV64-NEXT: [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
-; RV64-NEXT: [[TMP5:%.*]] = shl i64 [[TMP1]], 12
-; RV64-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP5]]
-; RV64-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP2]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
-; RV64-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
-; RV64-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; RV64-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; RV64-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[ARR]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP0]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV64-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; RV64: [[MIDDLE_BLOCK]]:
; RV64-NEXT: br label %[[EXIT:.*]]
; RV64: [[EXIT]]:
@@ -38,17 +29,9 @@ define void @narrow_iv_i8_sext_i64(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
-; RV32-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV32-NEXT: [[AVL:%.*]] = phi i32 [ 16, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
-; RV32-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 12
-; RV32-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i32 [[TMP1]]
-; RV32-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[TMP2]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
-; RV32-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
-; RV32-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; RV32-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; RV32-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[ARR]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP0]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV32-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; RV32: [[MIDDLE_BLOCK]]:
; RV32-NEXT: br label %[[EXIT:.*]]
; RV32: [[EXIT]]:
@@ -71,7 +54,7 @@ exit:
ret void
}
-define void @narrow_iv_i8_sext_i16(ptr noalias %arr, ptr noalias %out) {
+define void @narrow_iv_i8_sext_i16(ptr noalias %arr, ptr noalias %out) vscale_range(2, 1024) {
; RV64-LABEL: define void @narrow_iv_i8_sext_i16(
; RV64-SAME: ptr noalias [[ARR:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0]] {
; RV64-NEXT: [[ENTRY:.*:]]
@@ -81,18 +64,9 @@ define void @narrow_iv_i8_sext_i16(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
-; RV64-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV64-NEXT: [[AVL:%.*]] = phi i32 [ 16, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
-; RV64-NEXT: [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
-; RV64-NEXT: [[TMP5:%.*]] = shl i64 [[TMP1]], 12
-; RV64-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP5]]
-; RV64-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP2]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
-; RV64-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
-; RV64-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; RV64-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; RV64-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[ARR]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP0]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV64-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; RV64: [[MIDDLE_BLOCK]]:
; RV64-NEXT: br label %[[EXIT:.*]]
; RV64: [[EXIT]]:
@@ -107,17 +81,9 @@ define void @narrow_iv_i8_sext_i16(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
-; RV32-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV32-NEXT: [[AVL:%.*]] = phi i32 [ 16, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
-; RV32-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 12
-; RV32-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i32 [[TMP1]]
-; RV32-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[TMP2]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
-; RV32-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
-; RV32-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; RV32-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; RV32-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[ARR]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP0]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV32-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; RV32: [[MIDDLE_BLOCK]]:
; RV32-NEXT: br label %[[EXIT:.*]]
; RV32: [[EXIT]]:
@@ -140,7 +106,7 @@ exit:
ret void
}
-define void @narrow_iv_i8_zext_i64(ptr noalias %arr, ptr noalias %out) {
+define void @narrow_iv_i8_zext_i64(ptr noalias %arr, ptr noalias %out) vscale_range(2, 1024) {
; RV64-LABEL: define void @narrow_iv_i8_zext_i64(
; RV64-SAME: ptr noalias [[ARR:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0]] {
; RV64-NEXT: [[ENTRY:.*:]]
@@ -150,18 +116,9 @@ define void @narrow_iv_i8_zext_i64(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
-; RV64-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV64-NEXT: [[AVL:%.*]] = phi i32 [ 16, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
-; RV64-NEXT: [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
-; RV64-NEXT: [[TMP5:%.*]] = shl i64 [[TMP1]], 12
-; RV64-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP5]]
-; RV64-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP2]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
-; RV64-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
-; RV64-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; RV64-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; RV64-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[ARR]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP0]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV64-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; RV64: [[MIDDLE_BLOCK]]:
; RV64-NEXT: br label %[[EXIT:.*]]
; RV64: [[EXIT]]:
@@ -176,17 +133,9 @@ define void @narrow_iv_i8_zext_i64(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
-; RV32-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV32-NEXT: [[AVL:%.*]] = phi i32 [ 16, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
-; RV32-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 12
-; RV32-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i32 [[TMP1]]
-; RV32-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[TMP2]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
-; RV32-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
-; RV32-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; RV32-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; RV32-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[ARR]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP0]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV32-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; RV32: [[MIDDLE_BLOCK]]:
; RV32-NEXT: br label %[[EXIT:.*]]
; RV32: [[EXIT]]:
@@ -209,7 +158,7 @@ exit:
ret void
}
-define void @narrow_iv_i8_zext_i16(ptr noalias %arr, ptr noalias %out) {
+define void @narrow_iv_i8_zext_i16(ptr noalias %arr, ptr noalias %out) vscale_range(2, 1024) {
; RV64-LABEL: define void @narrow_iv_i8_zext_i16(
; RV64-SAME: ptr noalias [[ARR:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0]] {
; RV64-NEXT: [[ENTRY:.*:]]
@@ -219,18 +168,9 @@ define void @narrow_iv_i8_zext_i16(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
-; RV64-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV64-NEXT: [[AVL:%.*]] = phi i32 [ 16, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
-; RV64-NEXT: [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
-; RV64-NEXT: [[TMP5:%.*]] = shl i64 [[TMP1]], 12
-; RV64-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP5]]
-; RV64-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP2]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
-; RV64-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
-; RV64-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; RV64-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; RV64-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[ARR]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP0]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV64-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; RV64: [[MIDDLE_BLOCK]]:
; RV64-NEXT: br label %[[EXIT:.*]]
; RV64: [[EXIT]]:
@@ -245,17 +185,9 @@ define void @narrow_iv_i8_zext_i16(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
-; RV32-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV32-NEXT: [[AVL:%.*]] = phi i32 [ 16, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
-; RV32-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 12
-; RV32-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i32 [[TMP1]]
-; RV32-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[TMP2]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
-; RV32-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
-; RV32-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; RV32-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; RV32-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[ARR]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP0]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 16)
+; RV32-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; RV32: [[MIDDLE_BLOCK]]:
; RV32-NEXT: br label %[[EXIT:.*]]
; RV32: [[EXIT]]:
@@ -279,7 +211,7 @@ exit:
}
; Negative test
-define void @narrow_iv_i8_sext_i64_wrapping(ptr noalias %arr, ptr noalias %out) {
+define void @narrow_iv_i8_sext_i64_wrapping(ptr noalias %arr, ptr noalias %out) vscale_range(2, 1024) {
; RV64-LABEL: define void @narrow_iv_i8_sext_i64_wrapping(
; RV64-SAME: ptr noalias [[ARR:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0]] {
; RV64-NEXT: [[ENTRY:.*:]]
@@ -305,7 +237,7 @@ define void @narrow_iv_i8_sext_i64_wrapping(ptr noalias %arr, ptr noalias %out)
; RV64-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
; RV64-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; RV64-NEXT: [[TMP6:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; RV64-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]], !llvm.loop [[LOOP6:![0-9]+]]
+; RV64-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]], !llvm.loop [[LOOP0:![0-9]+]]
; RV64: [[MIDDLE_BLOCK]]:
; RV64-NEXT: br label %[[EXIT1:.*]]
; RV64: [[EXIT1]]:
@@ -336,7 +268,7 @@ define void @narrow_iv_i8_sext_i64_wrapping(ptr noalias %arr, ptr noalias %out)
; RV32-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
; RV32-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; RV32-NEXT: [[TMP6:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; RV32-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]], !llvm.loop [[LOOP6:![0-9]+]]
+; RV32-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]], !llvm.loop [[LOOP0:![0-9]+]]
; RV32: [[MIDDLE_BLOCK]]:
; RV32-NEXT: br label %[[EXIT1:.*]]
; RV32: [[EXIT1]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 714953ce460ed..ac02ef4bf2baa 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -5,7 +5,7 @@
; RUN: opt < %s -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -lv-strided-pointer-ivs=true -laa-speculate-unit-stride=false -force-vector-interleave=2 -S | FileCheck --check-prefixes=COMMON,STRIDED-COMMON,CHECK-UF2,STRIDED-UF2 %s
-define void @single_constant_stride_int_scaled(ptr %p) {
+define void @single_constant_stride_int_scaled(ptr %p) vscale_range(2, 1024) vscale_range(2, 1024) {
; CHECK-LABEL: define void @single_constant_stride_int_scaled(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -97,7 +97,7 @@ exit:
ret void
}
-define void @single_constant_stride_int_iv(ptr %p) {
+define void @single_constant_stride_int_iv(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: define void @single_constant_stride_int_iv(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -193,7 +193,7 @@ exit:
}
-define void @single_constant_stride_ptr_iv(ptr %p) {
+define void @single_constant_stride_ptr_iv(ptr %p) vscale_range(2, 1024) {
; CHECK-LABEL: define void @single_constant_stride_ptr_iv(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -299,7 +299,7 @@ exit:
}
-define void @single_stride_int_scaled(ptr %p, i64 %stride) {
+define void @single_stride_int_scaled(ptr %p, i64 %stride) vscale_range(2, 1024) {
; NOSTRIDED-LABEL: define void @single_stride_int_scaled(
; NOSTRIDED-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; NOSTRIDED-NEXT: [[ENTRY:.*:]]
@@ -310,7 +310,7 @@ define void @single_stride_int_scaled(ptr %p, i64 %stride) {
; NOSTRIDED: [[VECTOR_PH]]:
; NOSTRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; NOSTRIDED: [[VECTOR_BODY]]:
-; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; NOSTRIDED-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
@@ -318,7 +318,7 @@ define void @single_stride_int_scaled(ptr %p, i64 %stride) {
; NOSTRIDED-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 1)
; NOSTRIDED-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP10]], ptr align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; NOSTRIDED-NEXT: [[TMP11:%.*]] = zext i32 [[TMP7]] to i64
-; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; NOSTRIDED-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
; NOSTRIDED-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; NOSTRIDED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; NOSTRIDED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
@@ -421,7 +421,7 @@ exit:
ret void
}
-define void @single_stride_int_iv(ptr %p, i64 %stride) {
+define void @single_stride_int_iv(ptr %p, i64 %stride) vscale_range(2, 1024) {
; NOSTRIDED-LABEL: define void @single_stride_int_iv(
; NOSTRIDED-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; NOSTRIDED-NEXT: [[ENTRY:.*:]]
@@ -432,7 +432,7 @@ define void @single_stride_int_iv(ptr %p, i64 %stride) {
; NOSTRIDED: [[VECTOR_PH]]:
; NOSTRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; NOSTRIDED: [[VECTOR_BODY]]:
-; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; NOSTRIDED-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
@@ -440,7 +440,7 @@ define void @single_stride_int_iv(ptr %p, i64 %stride) {
; NOSTRIDED-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 1)
; NOSTRIDED-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP10]], ptr align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; NOSTRIDED-NEXT: [[TMP11:%.*]] = zext i32 [[TMP7]] to i64
-; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; NOSTRIDED-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
; NOSTRIDED-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; NOSTRIDED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; NOSTRIDED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
@@ -548,7 +548,7 @@ exit:
}
-define void @single_stride_ptr_iv(ptr %p, i64 %stride) {
+define void @single_stride_ptr_iv(ptr %p, i64 %stride) vscale_range(2, 1024) {
; COMMON-LABEL: define void @single_stride_ptr_iv(
; COMMON-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
; COMMON-NEXT: [[ENTRY:.*]]:
@@ -584,7 +584,7 @@ exit:
ret void
}
-define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
+define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) vscale_range(2, 1024) {
; NOSTRIDED-LABEL: define void @double_stride_int_scaled(
; NOSTRIDED-SAME: ptr [[P:%.*]], ptr [[P2:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; NOSTRIDED-NEXT: [[ENTRY:.*:]]
@@ -596,8 +596,8 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; NOSTRIDED-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NOSTRIDED: [[VECTOR_MEMCHECK]]:
; NOSTRIDED-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT: [[TMP1:%.*]] = shl i64 [[TMP3]], 4
-; NOSTRIDED-NEXT: [[TMP8:%.*]] = add i64 [[TMP1]], -1
+; NOSTRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; NOSTRIDED-NEXT: [[TMP8:%.*]] = add nsw i64 [[TMP1]], -1
; NOSTRIDED-NEXT: [[TMP6:%.*]] = sub i64 [[P21]], [[P3]]
; NOSTRIDED-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], 1
; NOSTRIDED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP8]]
@@ -605,7 +605,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; NOSTRIDED: [[VECTOR_PH]]:
; NOSTRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; NOSTRIDED: [[VECTOR_BODY]]:
-; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[TMP16:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; NOSTRIDED-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
@@ -614,7 +614,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; NOSTRIDED-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[P2]], i64 [[INDEX]]
; NOSTRIDED-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP14]], ptr align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
; NOSTRIDED-NEXT: [[TMP13:%.*]] = zext i32 [[TMP16]] to i64
-; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
+; NOSTRIDED-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
; NOSTRIDED-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; NOSTRIDED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; NOSTRIDED-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
@@ -643,16 +643,15 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
; NOSTRIDED-UF2-NEXT: [[P21:%.*]] = ptrtoaddr ptr [[P2]] to i64
; NOSTRIDED-UF2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NOSTRIDED-UF2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; NOSTRIDED-UF2-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 14)
-; NOSTRIDED-UF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; NOSTRIDED-UF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; NOSTRIDED-UF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; NOSTRIDED-UF2: [[VECTOR_SCEVCHECK]]:
; NOSTRIDED-UF2-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; NOSTRIDED-UF2-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; NOSTRIDED-UF2: [[VECTOR_MEMCHECK]]:
; NOSTRIDED-UF2-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-UF2-NEXT: [[TMP4:%.*]] = shl i64 [[TMP2]], 5
-; NOSTRIDED-UF2-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
+; NOSTRIDED-UF2-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 5
+; NOSTRIDED-UF2-NEXT: [[TMP15:%.*]] = add nsw i64 [[TMP3]], -1
; NOSTRIDED-UF2-NEXT: [[TMP5:%.*]] = sub i64 [[P21]], [[P3]]
; NOSTRIDED-UF2-NEXT: [[TMP10:%.*]] = sub i64 [[TMP5]], 1
; NOSTRIDED-UF2-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
@@ -912,7 +911,7 @@ exit:
ret void
}
-define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) {
+define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) vscale_range(2, 1024) {
; NOSTRIDED-LABEL: define void @double_stride_int_iv(
; NOSTRIDED-SAME: ptr [[P:%.*]], ptr [[P2:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; NOSTRIDED-NEXT: [[ENTRY:.*:]]
@@ -923,7 +922,7 @@ define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) {
; NOSTRIDED: [[VECTOR_PH]]:
; NOSTRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; NOSTRIDED: [[VECTOR_BODY]]:
-; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOSTRIDED-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; NOSTRIDED-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
@@ -931,7 +930,7 @@ define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) {
; NOSTRIDED-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 1)
; NOSTRIDED-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP10]], ptr align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; NOSTRIDED-NEXT: [[TMP11:%.*]] = zext i32 [[TMP7]] to i64
-; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; NOSTRIDED-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
; NOSTRIDED-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; NOSTRIDED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; NOSTRIDED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
@@ -1042,7 +1041,7 @@ exit:
ret void
}
-define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
+define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) vscale_range(2, 1024) {
; NOSTRIDED-LABEL: define void @double_stride_ptr_iv(
; NOSTRIDED-SAME: ptr [[P:%.*]], ptr [[P2:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; NOSTRIDED-NEXT: [[ENTRY:.*]]:
@@ -1262,7 +1261,7 @@ exit:
; }
; }
;
-define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) {
+define void @constant_stride_reinterpret(ptr noalias %in, ptr noalias %out) vscale_range(2, 1024) {
; CHECK-LABEL: define void @constant_stride_reinterpret(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -1354,7 +1353,7 @@ exit:
; Check that an access pattern matching as both an interleave group and a
; strided access at the same time is vectorized as an interleaved load rather
; than a strided load.
-define void @interleaved_load_instead_of_strided(ptr %a) {
+define void @interleaved_load_instead_of_strided(ptr %a) vscale_range(2, 1024) {
; CHECK-LABEL: define void @interleaved_load_instead_of_strided(
; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -1487,7 +1486,7 @@ exit:
}
; VPTypeAnalysis issue: crash in convertToStridedAccesses due to stale cache.
-define void @type_cache_crash(ptr noalias %a, ptr noalias %b, ptr noalias %c, i32 %n, double %c0, double %c1, double %c2, i64 %c3) {
+define void @type_cache_crash(ptr noalias %a, ptr noalias %b, ptr noalias %c, i32 %n, double %c0, double %c1, double %c2, i64 %c3) vscale_range(2, 1024) {
; CHECK-LABEL: define void @type_cache_crash(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i32 [[N:%.*]], double [[C0:%.*]], double [[C1:%.*]], double [[C2:%.*]], i64 [[C3:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -1644,7 +1643,7 @@ exit:
ret void
}
-define void @lshr_exact_stride(ptr noalias %in, ptr noalias %out) {
+define void @lshr_exact_stride(ptr noalias %in, ptr noalias %out) vscale_range(2, 1024) {
; CHECK-LABEL: define void @lshr_exact_stride(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -1748,7 +1747,7 @@ exit:
ret void
}
-define void @or_disjoint_stride(ptr noalias %in, ptr noalias %out) {
+define void @or_disjoint_stride(ptr noalias %in, ptr noalias %out) vscale_range(2, 1024) {
; CHECK-LABEL: define void @or_disjoint_stride(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -1851,7 +1850,7 @@ exit:
ret void
}
-define void @lshr_stride_no_exact(ptr noalias %in, ptr noalias %out) {
+define void @lshr_stride_no_exact(ptr noalias %in, ptr noalias %out) vscale_range(2, 1024) {
; CHECK-LABEL: define void @lshr_stride_no_exact(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -1955,7 +1954,7 @@ exit:
ret void
}
-define void @lshr_exact_stride_nonconstant(ptr noalias %in, ptr noalias %out, i64 %shift) {
+define void @lshr_exact_stride_nonconstant(ptr noalias %in, ptr noalias %out, i64 %shift) vscale_range(2, 1024) {
; CHECK-LABEL: define void @lshr_exact_stride_nonconstant(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias [[OUT:%.*]], i64 [[SHIFT:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -2071,7 +2070,7 @@ exit:
; The routine for creating strided accesses depends on SCEV expansion,
; and we specifically test the case of IV-start being an AddRec.
-define void @iv_start_is_addrec(ptr %p, ptr noalias %q) {
+define void @iv_start_is_addrec(ptr %p, ptr noalias %q) vscale_range(2, 1024) {
; CHECK-LABEL: define void @iv_start_is_addrec(
; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[Q:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
index 482fdeb61b505..56412b875695e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
@@ -8,7 +8,7 @@
; RUN: -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s --check-prefix=NO-VP
-define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_and(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -17,8 +17,8 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -26,7 +26,7 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -35,7 +35,7 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -63,13 +63,12 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -125,7 +124,7 @@ finish.loopexit:
ret void
}
-define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_or(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -134,8 +133,8 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -143,7 +142,7 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -152,7 +151,7 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -180,13 +179,12 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -242,7 +240,7 @@ finish.loopexit:
ret void
}
-define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_xor(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -251,8 +249,8 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -260,7 +258,7 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -269,7 +267,7 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -297,13 +295,12 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -359,7 +356,7 @@ finish.loopexit:
ret void
}
-define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_shl(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -368,8 +365,8 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -377,7 +374,7 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -386,7 +383,7 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -414,13 +411,12 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -476,7 +472,7 @@ finish.loopexit:
ret void
}
-define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_lshr(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -485,8 +481,8 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -494,7 +490,7 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -503,7 +499,7 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -531,13 +527,12 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -593,7 +588,7 @@ finish.loopexit:
ret void
}
-define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_ashr(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -602,8 +597,8 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -611,7 +606,7 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -620,7 +615,7 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -648,13 +643,12 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -710,7 +704,7 @@ finish.loopexit:
ret void
}
-define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_add(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -719,8 +713,8 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -728,7 +722,7 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -737,7 +731,7 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
@@ -765,13 +759,12 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -827,7 +820,7 @@ finish.loopexit:
ret void
}
-define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_sub(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -836,8 +829,8 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -845,7 +838,7 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -854,7 +847,7 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
@@ -882,13 +875,12 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -944,7 +936,7 @@ finish.loopexit:
ret void
}
-define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_mul(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -953,8 +945,8 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -962,7 +954,7 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -971,7 +963,7 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
@@ -999,13 +991,12 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -1061,7 +1052,7 @@ finish.loopexit:
ret void
}
-define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_sdiv(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -1070,8 +1061,8 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -1079,7 +1070,7 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -1088,7 +1079,7 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
@@ -1116,13 +1107,12 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -1178,7 +1168,7 @@ finish.loopexit:
ret void
}
-define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_udiv(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -1187,8 +1177,8 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -1196,7 +1186,7 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -1205,7 +1195,7 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
@@ -1233,13 +1223,12 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -1295,7 +1284,7 @@ finish.loopexit:
ret void
}
-define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_srem(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -1304,8 +1293,8 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -1313,7 +1302,7 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -1322,7 +1311,7 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
@@ -1350,13 +1339,12 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -1412,7 +1400,7 @@ finish.loopexit:
ret void
}
-define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_urem(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -1421,8 +1409,8 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP4]]
@@ -1430,7 +1418,7 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP10:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 16, i1 true)
; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -1439,7 +1427,7 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[VP_OP]], ptr align 1 [[TMP16]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; IF-EVL-NEXT: [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP18]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP10]], [[TMP18]]
; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
@@ -1467,13 +1455,12 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
-; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP5:%.*]] = shl i64 [[TMP2]], 4
-; NO-VP-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], -1
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 4
+; NO-VP-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP3]], -1
; NO-VP-NEXT: [[TMP4:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP8:%.*]] = sub i64 [[TMP4]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP13]]
@@ -1531,7 +1518,7 @@ finish.loopexit:
; Floating point tests
-define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_fadd(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -1540,8 +1527,8 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP5]]
@@ -1549,7 +1536,7 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP11]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -1558,7 +1545,7 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_OP]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP11]], [[TMP19]]
; IF-EVL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
@@ -1591,8 +1578,8 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add nsw i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
@@ -1648,7 +1635,7 @@ finish.loopexit:
ret void
}
-define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_fsub(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -1657,8 +1644,8 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP5]]
@@ -1666,7 +1653,7 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP11]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -1675,7 +1662,7 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_OP]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP11]], [[TMP19]]
; IF-EVL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
@@ -1708,8 +1695,8 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add nsw i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
@@ -1765,7 +1752,7 @@ finish.loopexit:
ret void
}
-define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_fmul(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -1774,8 +1761,8 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP5]]
@@ -1783,7 +1770,7 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP11]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -1792,7 +1779,7 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_OP]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP11]], [[TMP19]]
; IF-EVL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
@@ -1825,8 +1812,8 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add nsw i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
@@ -1882,7 +1869,7 @@ finish.loopexit:
ret void
}
-define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_fdiv(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -1891,8 +1878,8 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP5]]
@@ -1900,7 +1887,7 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP11]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -1909,7 +1896,7 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_OP]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP11]], [[TMP19]]
; IF-EVL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
@@ -1942,8 +1929,8 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add nsw i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
@@ -1999,7 +1986,7 @@ finish.loopexit:
ret void
}
-define void @test_frem(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_frem(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_frem(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*]]:
@@ -2052,7 +2039,7 @@ finish.loopexit:
ret void
}
-define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
+define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_fneg(
; IF-EVL-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -2061,8 +2048,8 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; IF-EVL: [[VECTOR_MEMCHECK]]:
; IF-EVL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4
-; IF-EVL-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], -1
+; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; IF-EVL-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP1]], -1
; IF-EVL-NEXT: [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
; IF-EVL-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], 1
; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP5]]
@@ -2070,7 +2057,7 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP11:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP11]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -2079,7 +2066,7 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_OP]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP19]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[TMP11]], [[TMP19]]
; IF-EVL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
@@ -2112,8 +2099,8 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 4
-; NO-VP-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], -1
+; NO-VP-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; NO-VP-NEXT: [[TMP15:%.*]] = add nsw i64 [[TMP4]], -1
; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
; NO-VP-NEXT: [[TMP10:%.*]] = sub i64 [[TMP6]], 1
; NO-VP-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[TMP15]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll
index 4374c756cffec..9e56b65ee8d89 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll
@@ -7,7 +7,7 @@
; RUN: -tail-folding-policy=dont-fold-tail \
; RUN: -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s --check-prefix=NO-VP
-define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_sdiv(
; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -15,7 +15,7 @@ define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -26,7 +26,7 @@ define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; IF-EVL-NEXT: [[TMP14:%.*]] = zext i32 [[TMP5]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
; IF-EVL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -40,8 +40,7 @@ define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:
; NO-VP-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP12]], i64 4)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP12]]
@@ -100,7 +99,7 @@ exit:
ret void
}
-define void @test_sdiv_divisor_invariant_nonconst(ptr noalias %a, i64 %b, ptr noalias %c) {
+define void @test_sdiv_divisor_invariant_nonconst(ptr noalias %a, i64 %b, ptr noalias %c) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_sdiv_divisor_invariant_nonconst(
; IF-EVL-SAME: ptr noalias [[A:%.*]], i64 [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -110,7 +109,7 @@ define void @test_sdiv_divisor_invariant_nonconst(ptr noalias %a, i64 %b, ptr no
; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; IF-EVL-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -119,7 +118,7 @@ define void @test_sdiv_divisor_invariant_nonconst(ptr noalias %a, i64 %b, ptr no
; IF-EVL-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], ptr align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]])
; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
; IF-EVL-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -133,8 +132,7 @@ define void @test_sdiv_divisor_invariant_nonconst(ptr noalias %a, i64 %b, ptr no
; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:
; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 4)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -189,7 +187,7 @@ exit:
ret void
}
-define void @test_sdiv_both_invariant_nonconst(ptr noalias %a, i64 %b, i64 %b2, ptr noalias %c) {
+define void @test_sdiv_both_invariant_nonconst(ptr noalias %a, i64 %b, i64 %b2, ptr noalias %c) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_sdiv_both_invariant_nonconst(
; IF-EVL-SAME: ptr noalias [[A:%.*]], i64 [[B:%.*]], i64 [[B2:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -201,7 +199,7 @@ define void @test_sdiv_both_invariant_nonconst(ptr noalias %a, i64 %b, i64 %b2,
; IF-EVL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; IF-EVL-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -211,7 +209,7 @@ define void @test_sdiv_both_invariant_nonconst(ptr noalias %a, i64 %b, i64 %b2,
; IF-EVL-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], ptr align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP1]])
; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP1]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
; IF-EVL-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -225,8 +223,7 @@ define void @test_sdiv_both_invariant_nonconst(ptr noalias %a, i64 %b, i64 %b2,
; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:
; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 4)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -284,7 +281,7 @@ exit:
ret void
}
-define void @test_sdiv_divisor_invariant_minusone(ptr noalias %a, ptr noalias %c) {
+define void @test_sdiv_divisor_invariant_minusone(ptr noalias %a, ptr noalias %c) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_sdiv_divisor_invariant_minusone(
; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -292,7 +289,7 @@ define void @test_sdiv_divisor_invariant_minusone(ptr noalias %a, ptr noalias %c
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; IF-EVL-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -301,7 +298,7 @@ define void @test_sdiv_divisor_invariant_minusone(ptr noalias %a, ptr noalias %c
; IF-EVL-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], ptr align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]])
; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
; IF-EVL-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -315,8 +312,7 @@ define void @test_sdiv_divisor_invariant_minusone(ptr noalias %a, ptr noalias %c
; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:
; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 4)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -369,7 +365,7 @@ exit:
ret void
}
-define void @test_sdiv_divisor_invariant_safeimm(ptr noalias %a, ptr noalias %c) {
+define void @test_sdiv_divisor_invariant_safeimm(ptr noalias %a, ptr noalias %c) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_sdiv_divisor_invariant_safeimm(
; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -377,7 +373,7 @@ define void @test_sdiv_divisor_invariant_safeimm(ptr noalias %a, ptr noalias %c)
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; IF-EVL-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -386,7 +382,7 @@ define void @test_sdiv_divisor_invariant_safeimm(ptr noalias %a, ptr noalias %c)
; IF-EVL-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP2]], ptr align 8 [[TMP3]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]])
; IF-EVL-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP4]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP4]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
; IF-EVL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -400,8 +396,7 @@ define void @test_sdiv_divisor_invariant_safeimm(ptr noalias %a, ptr noalias %c)
; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:
; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 4)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -454,7 +449,7 @@ exit:
ret void
}
-define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_udiv(
; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -462,7 +457,7 @@ define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -473,7 +468,7 @@ define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; IF-EVL-NEXT: [[TMP14:%.*]] = zext i32 [[TMP5]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
; IF-EVL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
@@ -487,8 +482,7 @@ define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:
; NO-VP-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP12]], i64 4)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP12]]
@@ -547,7 +541,7 @@ exit:
ret void
}
-define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_srem(
; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -555,7 +549,7 @@ define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -566,7 +560,7 @@ define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; IF-EVL-NEXT: [[TMP14:%.*]] = zext i32 [[TMP5]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
; IF-EVL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -580,8 +574,7 @@ define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:
; NO-VP-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP12]], i64 4)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP12]]
@@ -640,7 +633,7 @@ exit:
ret void
}
-define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) vscale_range(2, 1024) {
; IF-EVL-LABEL: define void @test_urem(
; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]
@@ -648,7 +641,7 @@ define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; IF-EVL: [[VECTOR_PH]]:
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]
@@ -659,7 +652,7 @@ define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; IF-EVL-NEXT: [[TMP14:%.*]] = zext i32 [[TMP5]] to i64
-; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
; IF-EVL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; IF-EVL-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
@@ -673,8 +666,7 @@ define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:
; NO-VP-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP12]], i64 4)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NO-VP: [[VECTOR_PH]]:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP12]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
index f51a863973cfc..4a7e037f04164 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
@@ -7,7 +7,7 @@
; RUN: -tail-folding-policy=dont-fold-tail \
; RUN: -mtriple=riscv64 -mattr=+v -S < %s | FileCheck --check-prefix=NO-VP %s
-define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
+define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) vscale_range(2, 1024) {
; IF-EVL-LABEL: @interleave(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -104,14 +104,14 @@ for.cond.cleanup:
; E.g.
; int (*a)[4];
; int rdx = 0;
-; for (int i = 0; i < n; i++) {
+; for (int i = 0; i < n; i++) vscale_range(2, 1024) {
; rdx += a[i][0];
; rdx += a[i][1];
; // No access a[i][2]
; rdx += a[i][3];
; }
;
-define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
+define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) vscale_range(2, 1024) {
; IF-EVL-LABEL: @load_factor_4_with_gap(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -148,8 +148,7 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
; NO-VP-NEXT: entry:
; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP2]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
@@ -223,14 +222,14 @@ exit:
; Interleaved group with gap but without tail gap
; E.g.
; int (*a)[4];
-; for (int i = 0; i < n; i++) {
+; for (int i = 0; i < n; i++) vscale_range(2, 1024) {
; a[i][0] = i;
; a[i][1] = i;
; // No access a[i][2]
; a[i][3] = i;
; }
;
-define void @store_factor_4_with_gap(i32 %n, ptr noalias %a) {
+define void @store_factor_4_with_gap(i32 %n, ptr noalias %a) vscale_range(2, 1024) {
; IF-EVL-LABEL: @store_factor_4_with_gap(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -268,8 +267,7 @@ define void @store_factor_4_with_gap(i32 %n, ptr noalias %a) {
; NO-VP-NEXT: entry:
; NO-VP-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; NO-VP-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 8)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP2]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP1]]
@@ -338,14 +336,14 @@ exit:
; E.g.
; int (*a)[4];
; int rdx = 0;
-; for (int i = 0; i < n; i++) {
+; for (int i = 0; i < n; i++) vscale_range(2, 1024) {
; rdx += a[i][0];
; rdx += a[i][1];
; rdx += a[i][2];
; // No access a[i][3]
; }
;
-define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
+define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) vscale_range(2, 1024) {
; IF-EVL-LABEL: @load_factor_4_with_tail_gap(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -460,14 +458,14 @@ exit:
; E.g.
; int (*a)[4];
; int rdx = 0;
-; for (int i = 0; i < n; i++) {
+; for (int i = 0; i < n; i++) vscale_range(2, 1024) {
; a[i][0] = i;
; a[i][1] = i;
; a[i][2] = i;
; // No access a[i][3]
; }
;
-define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) {
+define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) vscale_range(2, 1024) {
; IF-EVL-LABEL: @store_factor_4_with_tail_gap(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -505,8 +503,7 @@ define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) {
; NO-VP-NEXT: entry:
; NO-VP-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; NO-VP-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 8)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP2]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP1]]
@@ -571,7 +568,7 @@ exit:
ret void
}
-define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
+define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) vscale_range(2, 1024) {
; IF-EVL-LABEL: @load_factor_4_reverse(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1
@@ -625,8 +622,7 @@ define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
; NO-VP-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[TMP1]]
; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
-; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP4]], i64 8)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP5]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP4]]
@@ -717,7 +713,7 @@ exit:
ret i32 %add3
}
-define void @store_factor_4_reverse(i32 %n, ptr noalias %a) {
+define void @store_factor_4_reverse(i32 %n, ptr noalias %a) vscale_range(2, 1024) {
; IF-EVL-LABEL: @store_factor_4_reverse(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: [[TMP0:%.*]] = add nsw i32 [[N:%.*]], -1
@@ -774,8 +770,7 @@ define void @store_factor_4_reverse(i32 %n, ptr noalias %a) {
; NO-VP-NEXT: [[TMP2:%.*]] = sub i32 [[N]], [[TMP1]]
; NO-VP-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
; NO-VP-NEXT: [[TMP4:%.*]] = shl nuw i32 [[TMP3]], 2
-; NO-VP-NEXT: [[TMP5:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP4]], i32 8)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP2]], [[TMP5]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP2]], [[TMP4]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP2]], [[TMP4]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
index 229bfbae6df2d..b9dceefde871b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
@@ -7,7 +7,7 @@
; RUN: -tail-folding-policy=dont-fold-tail \
; RUN: -mtriple=riscv64 -mattr=+v -S < %s | FileCheck %s --check-prefix=NO-VP
-define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %ptr2) {
+define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %ptr2) vscale_range(2, 1024) {
; IF-EVL-LABEL: @reverse_load_store(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -41,8 +41,7 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
; NO-VP-NEXT: entry:
; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -105,7 +104,7 @@ loopend:
ret void
}
-define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noalias %ptr1, ptr noalias %ptr2) {
+define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noalias %ptr1, ptr noalias %ptr2) vscale_range(2, 1024) {
; IF-EVL-LABEL: @reverse_load_store_masked(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -144,8 +143,7 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
; NO-VP-NEXT: entry:
; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[ENTRY:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
@@ -232,7 +230,7 @@ loopend:
; From a miscompile originally reported at
; https://github.com/llvm/llvm-project/issues/122681
-define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %d) {
+define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %d) vscale_range(2, 1024) {
; IF-EVL-LABEL: @multiple_reverse_vector_pointer(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -272,8 +270,7 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
; NO-VP-NEXT: entry:
; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP5]], 4
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; NO-VP: vector.ph:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll
index 6896cfe5b152f..1f8f452e4ed36 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll
@@ -10,7 +10,7 @@
; Dependence distance between read and write is greater than the trip
; count of the loop. Thus, values written are never read for any
; valid vectorization of the loop.
-define void @test(ptr %p) {
+define void @test(ptr %p) vscale_range(8, 1024) {
; IF-EVL-LABEL: @test(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -39,8 +39,7 @@ define void @test(ptr %p) {
; NO-VP-NEXT: entry:
; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 4)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
; NO-VP: vector.ph:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 200, [[TMP1]]
@@ -95,7 +94,7 @@ exit:
; Dependence distance is less than trip count, thus we must prove that
; chosen VF guaranteed to be less than dependence distance.
-define void @test_may_clobber1(ptr %p) {
+define void @test_may_clobber1(ptr %p) vscale_range(2, 1024) {
; IF-EVL-LABEL: @test_may_clobber1(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -154,7 +153,7 @@ exit:
ret void
}
-define void @test_may_clobber2(ptr %p) {
+define void @test_may_clobber2(ptr %p) vscale_range(2, 1024) {
; IF-EVL-LABEL: @test_may_clobber2(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[LOOP:%.*]]
@@ -205,7 +204,7 @@ exit:
ret void
}
-define void @test_may_clobber3(ptr %p) {
+define void @test_may_clobber3(ptr %p) vscale_range(2, 1024) {
; IF-EVL-LABEL: @test_may_clobber3(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -265,7 +264,7 @@ exit:
}
; Trviailly no overlap due to maximum possible value of VLEN and LMUL
-define void @trivial_due_max_vscale(ptr %p) {
+define void @trivial_due_max_vscale(ptr %p) vscale_range(2, 1024) {
; IF-EVL-LABEL: @trivial_due_max_vscale(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -294,8 +293,7 @@ define void @trivial_due_max_vscale(ptr %p) {
; NO-VP-NEXT: entry:
; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 4)
-; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[UMAX]]
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[TMP1]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
; NO-VP: vector.ph:
; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 200, [[TMP1]]
@@ -349,7 +347,7 @@ exit:
}
; Dependence distance could be violated via LMUL>=2 or interleaving
-define void @no_high_lmul_or_interleave(ptr %p) {
+define void @no_high_lmul_or_interleave(ptr %p) vscale_range(2, 1024) {
; IF-EVL-LABEL: @no_high_lmul_or_interleave(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
@@ -426,7 +424,7 @@ exit:
ret void
}
-define void @non-power-2-storeloadforward(ptr %A) {
+define void @non-power-2-storeloadforward(ptr %A) vscale_range(2, 1024) {
; IF-EVL-LABEL: @non-power-2-storeloadforward(
; IF-EVL-NEXT: entry:
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
index f9914ff508423..b25b57169a6e6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
@@ -10,7 +10,7 @@ define void @load_store_interleave_group(ptr noalias %data) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; CHECK-NEXT: [[TMP1:%.*]] = shl nsw i64 [[EVL_BASED_IV]], 1
@@ -24,7 +24,7 @@ define void @load_store_interleave_group(ptr noalias %data) {
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]])
; CHECK-NEXT: call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr align 8 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL1]])
; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP5]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -244,7 +244,7 @@ define void @load_store_interleave_group_i32(ptr noalias %data) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[TMP1:%.*]] = shl nsw i64 [[EVL_BASED_IV]], 2
@@ -260,7 +260,7 @@ define void @load_store_interleave_group_i32(ptr noalias %data) {
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]])
; CHECK-NEXT: call void @llvm.vp.store.nxv16i32.p0(<vscale x 16 x i32> [[INTERLEAVED_VEC]], ptr align 8 [[TMP2]], <vscale x 16 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL1]])
; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP5]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index 1f02c3f4906b7..ae13f61c7a64d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -4,7 +4,7 @@
target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
target triple = "riscv64-unknown-linux-gnu"
-define void @test_pr98413_zext_removed(ptr %src, ptr noalias %dst, i64 %x) {
+define void @test_pr98413_zext_removed(ptr %src, ptr noalias %dst, i64 %x) vscale_range(2, 1024) {
; CHECK-LABEL: define void @test_pr98413_zext_removed(
; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -15,7 +15,7 @@ define void @test_pr98413_zext_removed(ptr %src, ptr noalias %dst, i64 %x) {
; CHECK-NEXT: [[TMP6:%.*]] = trunc <vscale x 8 x i64> [[BROADCAST_SPLAT]] to <vscale x 8 x i8>
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 97, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 [[TMP7]]
@@ -25,7 +25,7 @@ define void @test_pr98413_zext_removed(ptr %src, ptr noalias %dst, i64 %x) {
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP7]]
; CHECK-NEXT: call void @llvm.vp.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP11]], ptr align 1 [[TMP12]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
; CHECK-NEXT: [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[TMP7]]
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP14]], [[TMP7]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -54,7 +54,7 @@ exit:
ret void
}
-define void @test_pr98413_sext_removed(ptr %src, ptr noalias %dst, i64 %x) {
+define void @test_pr98413_sext_removed(ptr %src, ptr noalias %dst, i64 %x) vscale_range(2, 1024) {
; CHECK-LABEL: define void @test_pr98413_sext_removed(
; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -65,7 +65,7 @@ define void @test_pr98413_sext_removed(ptr %src, ptr noalias %dst, i64 %x) {
; CHECK-NEXT: [[TMP6:%.*]] = trunc <vscale x 8 x i64> [[BROADCAST_SPLAT]] to <vscale x 8 x i8>
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 97, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 [[TMP7]]
@@ -75,7 +75,7 @@ define void @test_pr98413_sext_removed(ptr %src, ptr noalias %dst, i64 %x) {
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP7]]
; CHECK-NEXT: call void @llvm.vp.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP11]], ptr align 1 [[TMP12]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
; CHECK-NEXT: [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[TMP7]]
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP14]], [[TMP7]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -247,7 +247,7 @@ exit:
}
; Test for https://github.com/llvm/llvm-project/issues/162688.
-define void @test_minbws_for_trunc(i32 %n, ptr noalias %p1, ptr noalias %p2) {
+define void @test_minbws_for_trunc(i32 %n, ptr noalias %p1, ptr noalias %p2) vscale_range(2, 1024) {
; CHECK-LABEL: define void @test_minbws_for_trunc(
; CHECK-SAME: i32 [[N:%.*]], ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
index 0d34b2b6d2d7b..57fff5324622b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
@@ -5,7 +5,7 @@
; an underlying value to an EVL recipe. This occurs in this test via
; VPlanTransforms::truncateToMinimalBitwidths
-define void @truncate_to_minimal_bitwidths_widen_cast_recipe(ptr %src) {
+define void @truncate_to_minimal_bitwidths_widen_cast_recipe(ptr %src) vscale_range(2, 1024) {
; CHECK-LABEL: define void @truncate_to_minimal_bitwidths_widen_cast_recipe(
; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -13,13 +13,8 @@ define void @truncate_to_minimal_bitwidths_widen_cast_recipe(ptr %src) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 9, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> zeroinitializer, <vscale x 8 x ptr> align 1 splat (ptr null), <vscale x 8 x i1> splat (i1 true), i32 [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP7]] to i64
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> zeroinitializer, <vscale x 8 x ptr> align 1 splat (ptr null), <vscale x 8 x i1> splat (i1 true), i32 9)
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -46,15 +41,14 @@ exit:
}
; Test case for https://github.com/llvm/llvm-project/issues/162374.
-define void @truncate_i16_to_i8_cse(ptr noalias %src, ptr noalias %dst) {
+define void @truncate_i16_to_i8_cse(ptr noalias %src, ptr noalias %dst) vscale_range(2, 1024) {
; CHECK-LABEL: define void @truncate_i16_to_i8_cse(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4294967296, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -70,25 +64,10 @@ define void @truncate_i16_to_i8_cse(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: store i8 [[TMP10]], ptr [[DST]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4294967296
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[COUNT:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[COUNT_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VAL:%.*]] = load i16, ptr [[SRC]], align 2
-; CHECK-NEXT: [[VAL_ZEXT:%.*]] = zext i16 [[VAL]] to i64
-; CHECK-NEXT: [[VAL_TRUNC_ZEXT:%.*]] = trunc i64 [[VAL_ZEXT]] to i8
-; CHECK-NEXT: store i8 [[VAL_TRUNC_ZEXT]], ptr null, align 1
-; CHECK-NEXT: [[VAL_TRUNC:%.*]] = trunc i16 [[VAL]] to i8
-; CHECK-NEXT: store i8 [[VAL_TRUNC]], ptr [[DST]], align 1
-; CHECK-NEXT: [[COUNT_NEXT]] = add i32 [[COUNT]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[COUNT_NEXT]], 0
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
@@ -116,6 +95,4 @@ exit:
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
index c7584291766a8..92d7b003c2128 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
@@ -6,7 +6,7 @@
target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
target triple = "riscv64"
-define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) {
+define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) vscale_range(2, 1024) {
; SCALABLE-LABEL: define void @uniform_load(
; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; SCALABLE-NEXT: [[ENTRY:.*:]]
@@ -14,7 +14,7 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6
; SCALABLE: [[VECTOR_PH]]:
; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; SCALABLE: [[VECTOR_BODY]]:
-; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; SCALABLE-NEXT: [[TMP7:%.*]] = load i64, ptr [[B]], align 8
@@ -23,7 +23,7 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6
; SCALABLE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; SCALABLE-NEXT: [[TMP11:%.*]] = zext i32 [[TMP10]] to i64
-; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; SCALABLE-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; SCALABLE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -72,7 +72,7 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6
; TF-SCALABLE: [[VECTOR_PH]]:
; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; TF-SCALABLE: [[VECTOR_BODY]]:
-; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; TF-SCALABLE-NEXT: [[TMP5:%.*]] = load i64, ptr [[B]], align 8
@@ -81,7 +81,7 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6
; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP6]])
; TF-SCALABLE-NEXT: [[TMP8:%.*]] = zext i32 [[TMP6]] to i64
-; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
+; TF-SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; TF-SCALABLE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; TF-SCALABLE-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -106,7 +106,7 @@ for.end:
ret void
}
-define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) {
+define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) vscale_range(2, 1024) {
; SCALABLE-LABEL: define i64 @uniform_load_outside_use(
; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; SCALABLE-NEXT: [[ENTRY:.*:]]
@@ -114,7 +114,7 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
; SCALABLE: [[VECTOR_PH]]:
; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; SCALABLE: [[VECTOR_BODY]]:
-; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; SCALABLE-NEXT: [[TMP6:%.*]] = load i64, ptr [[B]], align 8
@@ -123,7 +123,7 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
; SCALABLE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]])
; SCALABLE-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
-; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[INDEX]]
+; SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP5]], [[INDEX]]
; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
; SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; SCALABLE-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -173,7 +173,7 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
; TF-SCALABLE: [[VECTOR_PH]]:
; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; TF-SCALABLE: [[VECTOR_BODY]]:
-; TF-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; TF-SCALABLE-NEXT: [[V:%.*]] = load i64, ptr [[B]], align 8
@@ -182,7 +182,7 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
; TF-SCALABLE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[ARRAYIDX]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]])
; TF-SCALABLE-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
-; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[IV]]
+; TF-SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP5]], [[IV]]
; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
; TF-SCALABLE-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; TF-SCALABLE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -208,7 +208,7 @@ for.end:
}
-define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) {
+define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) vscale_range(2, 1024) {
; SCALABLE-LABEL: define void @conditional_uniform_load(
; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; SCALABLE-NEXT: [[ENTRY:.*:]]
@@ -219,7 +219,7 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
; SCALABLE-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; SCALABLE: [[VECTOR_BODY]]:
-; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[TMP17:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
@@ -231,7 +231,7 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
; SCALABLE-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], <vscale x 4 x i64> zeroinitializer
; SCALABLE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; SCALABLE-NEXT: call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[PREDPHI]], ptr align 8 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP17]])
-; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
+; SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
; SCALABLE-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -298,7 +298,7 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; TF-SCALABLE: [[VECTOR_BODY]]:
-; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
@@ -310,7 +310,7 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
; TF-SCALABLE-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], <vscale x 4 x i64> zeroinitializer
; TF-SCALABLE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[PREDPHI]], ptr align 8 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
-; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; TF-SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; TF-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
; TF-SCALABLE-NEXT: [[TMP17:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -343,7 +343,7 @@ for.end:
ret void
}
-define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) {
+define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) vscale_range(2, 1024) {
; SCALABLE-LABEL: define void @uniform_load_unaligned(
; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; SCALABLE-NEXT: [[ENTRY:.*:]]
@@ -351,7 +351,7 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt
; SCALABLE: [[VECTOR_PH]]:
; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; SCALABLE: [[VECTOR_BODY]]:
-; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; SCALABLE-NEXT: [[TMP6:%.*]] = load i64, ptr [[B]], align 1
@@ -360,7 +360,7 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt
; SCALABLE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; SCALABLE-NEXT: [[TMP11:%.*]] = zext i32 [[TMP10]] to i64
-; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; SCALABLE-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; SCALABLE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -409,7 +409,7 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt
; TF-SCALABLE: [[VECTOR_PH]]:
; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; TF-SCALABLE: [[VECTOR_BODY]]:
-; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; TF-SCALABLE-NEXT: [[TMP5:%.*]] = load i64, ptr [[B]], align 1
@@ -418,7 +418,7 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt
; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP6]])
; TF-SCALABLE-NEXT: [[TMP8:%.*]] = zext i32 [[TMP6]] to i64
-; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
+; TF-SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; TF-SCALABLE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; TF-SCALABLE-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -443,7 +443,7 @@ for.end:
ret void
}
-define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
+define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) vscale_range(2, 1024) {
; SCALABLE-LABEL: define void @uniform_store(
; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; SCALABLE-NEXT: [[ENTRY:.*:]]
@@ -453,14 +453,14 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; SCALABLE: [[VECTOR_BODY]]:
-; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; SCALABLE-NEXT: store i64 [[V]], ptr [[B]], align 8
; SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
; SCALABLE-NEXT: [[TMP10:%.*]] = zext i32 [[TMP9]] to i64
-; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
+; SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; SCALABLE-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; SCALABLE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -511,14 +511,14 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; TF-SCALABLE: [[VECTOR_BODY]]:
-; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; TF-SCALABLE-NEXT: store i64 [[V]], ptr [[B]], align 8
; TF-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP6]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; TF-SCALABLE-NEXT: [[TMP7:%.*]] = zext i32 [[TMP5]] to i64
-; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
+; TF-SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; TF-SCALABLE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; TF-SCALABLE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -543,7 +543,7 @@ for.end:
ret void
}
-define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
+define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) vscale_range(2, 1024) {
; SCALABLE-LABEL: define void @uniform_store_of_loop_varying(
; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; SCALABLE-NEXT: [[ENTRY:.*:]]
@@ -556,7 +556,7 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
; SCALABLE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; SCALABLE: [[VECTOR_BODY]]:
-; SCALABLE-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
@@ -566,7 +566,7 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
; SCALABLE-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT1]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; SCALABLE-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
; SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP16]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
-; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[TMP10]]
+; SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP8]], [[TMP10]]
; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
; SCALABLE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -627,7 +627,7 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; TF-SCALABLE: [[VECTOR_BODY]]:
-; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
@@ -637,7 +637,7 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
; TF-SCALABLE-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
; TF-SCALABLE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT3]], ptr align 8 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
-; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
+; TF-SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; TF-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; TF-SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -663,7 +663,7 @@ for.end:
ret void
}
-define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
+define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) vscale_range(2, 1024) {
; SCALABLE-LABEL: define void @conditional_uniform_store(
; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; SCALABLE-NEXT: [[ENTRY:.*:]]
@@ -676,7 +676,7 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
; SCALABLE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; SCALABLE: [[VECTOR_BODY]]:
-; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
@@ -687,7 +687,7 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
; SCALABLE-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[BROADCAST_SPLAT1]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT2]], <vscale x 2 x i1> [[TMP10]], i32 [[TMP7]])
; SCALABLE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT1]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
-; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[INDEX]]
+; SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP14]], [[INDEX]]
; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
; SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; SCALABLE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -755,7 +755,7 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; TF-SCALABLE: [[VECTOR_BODY]]:
-; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
@@ -766,7 +766,7 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
; TF-SCALABLE-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[BROADCAST_SPLAT1]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT2]], <vscale x 2 x i1> [[TMP10]], i32 [[TMP9]])
; TF-SCALABLE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT1]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
-; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; TF-SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; TF-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; TF-SCALABLE-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
@@ -798,7 +798,7 @@ for.end:
}
-define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
+define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) vscale_range(2, 1024) {
; SCALABLE-LABEL: define void @uniform_store_unaligned(
; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; SCALABLE-NEXT: [[ENTRY:.*:]]
@@ -808,14 +808,14 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap
; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; SCALABLE: [[VECTOR_BODY]]:
-; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; SCALABLE-NEXT: store i64 [[V]], ptr [[B]], align 1
; SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
; SCALABLE-NEXT: [[TMP10:%.*]] = zext i32 [[TMP9]] to i64
-; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
+; SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; SCALABLE-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; SCALABLE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
@@ -866,14 +866,14 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap
; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; TF-SCALABLE: [[VECTOR_BODY]]:
-; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; TF-SCALABLE-NEXT: store i64 [[V]], ptr [[B]], align 1
; TF-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP6]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; TF-SCALABLE-NEXT: [[TMP7:%.*]] = zext i32 [[TMP5]] to i64
-; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
+; TF-SCALABLE-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; TF-SCALABLE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; TF-SCALABLE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/zvl1024b-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/zvl1024b-crash.ll
index fcdea54e50b55..a36fe60d83b02 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/zvl1024b-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/zvl1024b-crash.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -p loop-vectorize -mtriple riscv64 -mattr=+v,+zvl1024b < %s -S | FileCheck %s
-define void @foo(i32 %0, ptr %p) {
+define void @foo(i32 %0, ptr %p) vscale_range(32, 1024) {
; CHECK-LABEL: define void @foo(
; CHECK-SAME: i32 [[TMP0:%.*]], ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
index e971ff766b053..c636351d89960 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "ir-bb<for.exit>" --version 6
; RUN: opt -mtriple riscv64 -passes=loop-vectorize -force-vector-width=4 -scalable-vectorization=on -mattr=+v,+experimental-zvdot4a8i -disable-output 2>&1 -vplan-print-after=optimizeEVLMasks < %s | FileCheck %s
-define i32 @vdota4(ptr %a, ptr %b) #0 {
+define i32 @vdota4(ptr %a, ptr %b) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'vdota4'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
index 2daf7496a618e..7bab46500720a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
@@ -3,7 +3,7 @@
; Tests for printing VPlans with reductions.
-define float @print_reduction(i64 %n, ptr noalias %y) {
+define float @print_reduction(i64 %n, ptr noalias %y) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_reduction'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -61,7 +61,7 @@ exit: ; preds = %loop, %entry
ret float %red.next
}
-define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr noalias %dst) {
+define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr noalias %dst) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_reduction_with_invariant_store'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -121,7 +121,7 @@ exit: ; preds = %loop, %entry
ret void
}
-define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
+define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_fmuladd_strict'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -185,7 +185,7 @@ exit:
ret float %muladd
}
-define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
+define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'find_last_iv'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -247,7 +247,7 @@ exit:
ret i64 %cond
}
-define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) {
+define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_extended_reduction'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -305,7 +305,7 @@ exit:
ret i64 %r.0.lcssa
}
-define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) {
+define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_mulacc'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -369,7 +369,7 @@ exit:
ret i64 %r.0.lcssa
}
-define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) {
+define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_mulacc_extended'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -438,7 +438,7 @@ exit:
ret i64 %r.0.lcssa
}
-define i32 @print_mulacc_negated(ptr %a, ptr %b) {
+define i32 @print_mulacc_negated(ptr %a, ptr %b) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_mulacc_negated'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -509,7 +509,7 @@ exit:
ret i32 %add
}
-define i32 @print_mulacc_extended_const(ptr %start, ptr %end) {
+define i32 @print_mulacc_extended_const(ptr %start, ptr %end) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_mulacc_extended_const'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -571,7 +571,7 @@ exit:
ret i32 %red.next
}
-define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) {
+define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_mulacc_extended_const_lhs'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -634,7 +634,7 @@ exit:
}
; Constants >= 128 cannot be treated as sign-extended, so the expression shouldn't extend 128
-define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) {
+define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_mulacc_not_extended_const'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -697,7 +697,7 @@ exit:
ret i32 %red.next.lcssa
}
-define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
+define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_ext_mulacc_extended_const'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -761,7 +761,7 @@ exit:
}
; Constants >= 128 cannot be treated as sign-extended, so the expression shouldn't extend 128
-define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
+define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_ext_mulacc_not_extended_const'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -829,7 +829,7 @@ exit:
; ExtMulAccReduction VPExpressionRecipe since the mul has two users.
; It can however be turned into an ExtendedReduction since that one doesn't
; modify the mul's operands.
-define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
+define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_ext_mul_two_uses'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
@@ -891,7 +891,7 @@ exit:
ret i64 %add
}
-define i32 @print_umax_reduction(ptr %y) {
+define i32 @print_umax_reduction(ptr %y) vscale_range(2, 1024) {
; CHECK-LABEL: VPlan for loop in 'print_umax_reduction'
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics.ll
index c3fc061affb55..1542baf21a985 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics.ll
@@ -1,79 +1,87 @@
-; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block" --version 6
+; RUN: opt -passes=loop-vectorize -vplan-print-after=printOptimizedVPlan at 2 \
; RUN: -tail-folding-policy=prefer-fold-tail \
-; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=IF-EVL,CHECK %s
+; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output %s 2>&1 | FileCheck --check-prefixes=IF-EVL %s
-; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \
+; RUN: opt -passes=loop-vectorize -vplan-print-after=printOptimizedVPlan at 2 \
; RUN: -tail-folding-policy=dont-fold-tail \
-; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=NO-VP,CHECK %s
+; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output %s 2>&1 | FileCheck --check-prefixes=NO-VP %s
-define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
-; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
-; IF-EVL-NOT: CURRENT-ITERATION-PHI
-;
-; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
-; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
-; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
+define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) vscale_range(2, 1024) {
+; IF-EVL-LABEL: VPlan for loop in 'foo'
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
; IF-EVL-EMPTY:
-; IF-EVL: vector.ph:
-; IF-EVL-NEXT: Successor(s): vector loop
+; IF-EVL-NEXT: ir-bb<entry>:
+; IF-EVL-NEXT: Successor(s): scalar.ph, vector.ph
; IF-EVL-EMPTY:
-; IF-EVL-NEXT: <x1> vector loop: {
-; IF-EVL-NEXT: vp<[[IV:%[0-9]+]]> = CANONICAL-IV
+; IF-EVL-NEXT: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
; IF-EVL-EMPTY:
-; IF-EVL-NEXT: vector.body:
-; IF-EVL-NEXT: CURRENT-ITERATION-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
-; IF-EVL-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%N>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
-; IF-EVL-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<%evl> to i64
-; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[CAST]]>
-; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
-; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds i32, ir<[[GEP1]]>
-; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<%evl>
-; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]>
-; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer inbounds i32, ir<[[GEP2]]>
-; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<%evl>
-; IF-EVL-NEXT: WIDEN ir<[[ADD:%.+]]> = add nsw ir<[[LD2]]>, ir<[[LD1]]>
-; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
-; IF-EVL-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer inbounds i32, ir<[[GEP3]]>
-; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<%evl>
-; IF-EVL-NEXT: EMIT-SCALAR vp<[[CAST2:%[0-9]+]]> = zext vp<%evl> to i64
-; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST2]]>, vp<[[EVL_PHI]]>
-; IF-EVL-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST2]]>
-; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
-; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
-; IF-EVL-NEXT: No successors
-; IF-EVL-NEXT: }
-
-; NO-VP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' {
-; NO-VP-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
-; NO-VP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
-; NO-VP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
-; NO-VP-NEXT: Live-in ir<%N> = original trip-count
+; IF-EVL-NEXT: <x1> vector loop: {
+; IF-EVL-NEXT: vp<[[VP2:%[0-9]+]]> = CANONICAL-IV
+; IF-EVL-EMPTY:
+; IF-EVL-NEXT: vector.body:
+; IF-EVL-NEXT: CURRENT-ITERATION-PHI vp<[[VP4:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; IF-EVL-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%N>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; IF-EVL-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; IF-EVL-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = zext vp<%evl> to i64
+; IF-EVL-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP5]]>
+; IF-EVL-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<[[VP6]]>
+; IF-EVL-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i32, ir<%arrayidx>, ir<1>
+; IF-EVL-NEXT: WIDEN ir<%0> = vp.load vp<[[VP7]]>, vp<%evl>
+; IF-EVL-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%c>, vp<[[VP6]]>
+; IF-EVL-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds i32, ir<%arrayidx2>, ir<1>
+; IF-EVL-NEXT: WIDEN ir<%1> = vp.load vp<[[VP8]]>, vp<%evl>
+; IF-EVL-NEXT: WIDEN ir<%add> = add nsw ir<%1>, ir<%0>
+; IF-EVL-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<[[VP6]]>
+; IF-EVL-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds i32, ir<%arrayidx4>, ir<1>
+; IF-EVL-NEXT: WIDEN vp.store vp<[[VP9]]>, ir<%add>, vp<%evl>
+; IF-EVL-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64
+; IF-EVL-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP10]]>, vp<[[VP4]]>
+; IF-EVL-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]>
+; IF-EVL-NEXT: EMIT vp<%index.next> = add vp<[[VP2]]>, vp<[[VP0]]>
+; IF-EVL-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; IF-EVL-NEXT: No successors
+; IF-EVL-NEXT: }
+; IF-EVL-NEXT: Successor(s): middle.block
+;
+; NO-VP-LABEL: VPlan for loop in 'foo'
+; NO-VP: VPlan 'Initial VPlan for VF={2,4,8},UF>=1' {
+; NO-VP-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; NO-VP-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; NO-VP-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; NO-VP-NEXT: Live-in ir<%N> = original trip-count
; NO-VP-EMPTY:
-; NO-VP: vector.ph:
-; NO-VP-NEXT: Successor(s): vector loop
+; NO-VP-NEXT: ir-bb<entry>:
+; NO-VP-NEXT: Successor(s): scalar.ph, vector.ph
; NO-VP-EMPTY:
-; NO-VP-NEXT: <x1> vector loop: {
-; NO-VP-NEXT: vp<[[IV:%[0-9]+]]> = CANONICAL-IV
+; NO-VP-NEXT: vector.ph:
+; NO-VP-NEXT: Successor(s): vector loop
; NO-VP-EMPTY:
-; NO-VP-NEXT: vector.body:
-; NO-VP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
-; NO-VP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
-; NO-VP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds i32, ir<[[GEP1]]>
-; NO-VP-NEXT: WIDEN ir<[[LD1:%.+]]> = load vp<[[PTR1]]>
-; NO-VP-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]>
-; NO-VP-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer inbounds i32, ir<[[GEP2]]>
-; NO-VP-NEXT: WIDEN ir<[[LD2:%.+]]> = load vp<[[PTR2]]>
-; NO-VP-NEXT: WIDEN ir<[[ADD:%.+]]> = add nsw ir<[[LD2]]>, ir<[[LD1]]>
-; NO-VP-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
-; NO-VP-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer inbounds i32, ir<[[GEP3]]>
-; NO-VP-NEXT: WIDEN store vp<[[PTR3]]>, ir<[[ADD]]>
-; NO-VP-NEXT: EMIT vp<[[IV_NEXT:%.+]]> = add nuw vp<[[IV]]>, vp<[[VFUF]]>
-; NO-VP-NEXT: EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>
-; NO-VP-NEXT: No successors
-; NO-VP-NEXT: }
-
+; NO-VP-NEXT: <x1> vector loop: {
+; NO-VP-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; NO-VP-EMPTY:
+; NO-VP-NEXT: vector.body:
+; NO-VP-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; NO-VP-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<[[VP4]]>
+; NO-VP-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%arrayidx>, ir<1>
+; NO-VP-NEXT: WIDEN ir<%0> = load vp<[[VP5]]>
+; NO-VP-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%c>, vp<[[VP4]]>
+; NO-VP-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds i32, ir<%arrayidx2>, ir<1>
+; NO-VP-NEXT: WIDEN ir<%1> = load vp<[[VP6]]>
+; NO-VP-NEXT: WIDEN ir<%add> = add nsw ir<%1>, ir<%0>
+; NO-VP-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
+; NO-VP-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i32, ir<%arrayidx4>, ir<1>
+; NO-VP-NEXT: WIDEN store vp<[[VP7]]>, ir<%add>
+; NO-VP-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; NO-VP-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; NO-VP-NEXT: No successors
+; NO-VP-NEXT: }
+; NO-VP-NEXT: Successor(s): middle.block
+;
entry:
br label %for.body
@@ -94,33 +102,69 @@ for.cond.cleanup:
ret void
}
-define void @safe_dep(ptr %p) {
-; CHECK: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
-; CHECK-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
-; CHECK-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: Live-in ir<512> = original trip-count
-; CHECK-EMPTY:
-; CHECK: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
-; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[IV:%[0-9]+]]> = CANONICAL-IV
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
-; CHECK-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr ir<%p>, vp<[[ST]]>
-; CHECK-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer i64, ir<[[GEP1]]>
-; CHECK-NEXT: WIDEN ir<[[V:%.+]]> = load vp<[[PTR1]]>
-; CHECK-NEXT: CLONE ir<[[OFFSET:.+]]> = add vp<[[ST]]>, ir<100>
-; CHECK-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr ir<%p>, ir<[[OFFSET]]>
-; CHECK-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer i64, ir<[[GEP2]]>
-; CHECK-NEXT: WIDEN store vp<[[PTR2]]>, ir<[[V]]>
-; CHECK-NEXT: EMIT vp<[[IV_NEXT:%.+]]> = add nuw vp<[[IV]]>, vp<[[VFUF]]>
-; CHECK-NEXT: EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-
+define void @safe_dep(ptr %p) vscale_range(2, 1024) {
+; IF-EVL-LABEL: VPlan for loop in 'safe_dep'
+; IF-EVL: VPlan 'Initial VPlan for VF={2,4},UF>=1' {
+; IF-EVL-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; IF-EVL-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<512> = original trip-count
+; IF-EVL-EMPTY:
+; IF-EVL-NEXT: ir-bb<entry>:
+; IF-EVL-NEXT: Successor(s): scalar.ph, vector.ph
+; IF-EVL-EMPTY:
+; IF-EVL-NEXT: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
+; IF-EVL-EMPTY:
+; IF-EVL-NEXT: <x1> vector loop: {
+; IF-EVL-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; IF-EVL-EMPTY:
+; IF-EVL-NEXT: vector.body:
+; IF-EVL-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; IF-EVL-NEXT: CLONE ir<%a1> = getelementptr ir<%p>, vp<[[VP4]]>
+; IF-EVL-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer i64, ir<%a1>, ir<1>
+; IF-EVL-NEXT: WIDEN ir<%v> = load vp<[[VP5]]>
+; IF-EVL-NEXT: CLONE ir<%offset> = add vp<[[VP4]]>, ir<100>
+; IF-EVL-NEXT: CLONE ir<%a2> = getelementptr ir<%p>, ir<%offset>
+; IF-EVL-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer i64, ir<%a2>, ir<1>
+; IF-EVL-NEXT: WIDEN store vp<[[VP6]]>, ir<%v>
+; IF-EVL-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; IF-EVL-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; IF-EVL-NEXT: No successors
+; IF-EVL-NEXT: }
+; IF-EVL-NEXT: Successor(s): middle.block
+;
+; NO-VP-LABEL: VPlan for loop in 'safe_dep'
+; NO-VP: VPlan 'Initial VPlan for VF={2,4},UF>=1' {
+; NO-VP-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; NO-VP-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; NO-VP-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; NO-VP-NEXT: Live-in ir<512> = original trip-count
+; NO-VP-EMPTY:
+; NO-VP-NEXT: ir-bb<entry>:
+; NO-VP-NEXT: Successor(s): scalar.ph, vector.ph
+; NO-VP-EMPTY:
+; NO-VP-NEXT: vector.ph:
+; NO-VP-NEXT: Successor(s): vector loop
+; NO-VP-EMPTY:
+; NO-VP-NEXT: <x1> vector loop: {
+; NO-VP-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; NO-VP-EMPTY:
+; NO-VP-NEXT: vector.body:
+; NO-VP-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; NO-VP-NEXT: CLONE ir<%a1> = getelementptr ir<%p>, vp<[[VP4]]>
+; NO-VP-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer i64, ir<%a1>, ir<1>
+; NO-VP-NEXT: WIDEN ir<%v> = load vp<[[VP5]]>
+; NO-VP-NEXT: CLONE ir<%offset> = add vp<[[VP4]]>, ir<100>
+; NO-VP-NEXT: CLONE ir<%a2> = getelementptr ir<%p>, ir<%offset>
+; NO-VP-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer i64, ir<%a2>, ir<1>
+; NO-VP-NEXT: WIDEN store vp<[[VP6]]>, ir<%v>
+; NO-VP-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; NO-VP-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; NO-VP-NEXT: No successors
+; NO-VP-NEXT: }
+; NO-VP-NEXT: Successor(s): middle.block
+;
entry:
br label %loop
@@ -138,4 +182,3 @@ loop:
exit:
ret void
}
-
diff --git a/llvm/test/Transforms/PhaseOrdering/RISCV/any-of-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/RISCV/any-of-vectorization.ll
index fe25b1abd41f3..95416d5ea9cb8 100644
--- a/llvm/test/Transforms/PhaseOrdering/RISCV/any-of-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/RISCV/any-of-vectorization.ll
@@ -3,7 +3,7 @@
; Make sure we widen the vp.merge from LoopVectorizer to i8 in RISCVCodeGenPrepare
-define i32 @f(ptr %p, i32 %n) {
+define i32 @f(ptr %p, i32 %n) vscale_range(2, 1024) {
; CHECK-LABEL: define range(i32 0, 2) i32 @f(
; CHECK-SAME: ptr nofree readonly captures(none) [[P:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll b/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
index baf7bf38f3351..1693b232b4e2a 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
@@ -7,7 +7,7 @@
; innermost loop headers have a greater than or equal to frequency than any
; block it dominates.
-define void @f(i1 %x) !prof !0 {
+define void @f(i1 %x) vscale_range(2, 1024) !prof !0 {
; CHECK-LABEL: define void @f(
; CHECK-SAME: i1 [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {{.*}}{
; CHECK-NEXT: [[ENTRY:.*]]:
More information about the llvm-commits
mailing list