[llvm] [LV] Convert scatter w/uniform addr and mask being header mask to scalar store. (PR #172799)
Elvis Wang via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 22:45:11 PDT 2026
https://github.com/ElvisWang123 updated https://github.com/llvm/llvm-project/pull/172799
>From 1a8c0ae5fcef44d5b65ed531ec813be122cf022a Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 14 Apr 2026 19:27:52 -0700
Subject: [PATCH 1/6] Precommit test.
---
.../LoopVectorize/RISCV/uniform-load-store.ll | 185 ++++++++++++++++++
1 file changed, 185 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
index c7584291766a8..e068bf50506f4 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
@@ -898,6 +898,191 @@ for.end:
ret void
}
+define void @uniform_load_store(ptr %p, ptr %q, i32 %n) {
+; SCALABLE-LABEL: define void @uniform_load_store(
+; SCALABLE-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; SCALABLE-NEXT: [[ENTRY:.*:]]
+; SCALABLE-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; SCALABLE: [[VECTOR_SCEVCHECK]]:
+; SCALABLE-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; SCALABLE-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
+; SCALABLE-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; SCALABLE: [[VECTOR_MEMCHECK]]:
+; SCALABLE-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[Q]], i64 4
+; SCALABLE-NEXT: [[TMP2:%.*]] = add i32 [[N]], -1
+; SCALABLE-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; SCALABLE-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; SCALABLE-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 4
+; SCALABLE-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; SCALABLE-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP1]]
+; SCALABLE-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[P]], [[SCEVGEP]]
+; SCALABLE-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; SCALABLE-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; SCALABLE: [[VECTOR_PH]]:
+; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[Q]], i64 0
+; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; SCALABLE: [[VECTOR_BODY]]:
+; SCALABLE-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[AVL:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[EVL_BASED_IV]]
+; SCALABLE-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META10:![0-9]+]]
+; SCALABLE-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 1)
+; SCALABLE-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META10]]
+; SCALABLE-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP8]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META13:![0-9]+]], !noalias [[META10]]
+; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add i32 [[TMP6]], [[EVL_BASED_IV]]
+; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
+; SCALABLE-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; SCALABLE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; SCALABLE: [[MIDDLE_BLOCK]]:
+; SCALABLE-NEXT: br label %[[EXIT:.*]]
+; SCALABLE: [[SCALAR_PH]]:
+; SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; SCALABLE-NEXT: br label %[[LOOP:.*]]
+; SCALABLE: [[LOOP]]:
+; SCALABLE-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; SCALABLE-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
+; SCALABLE-NEXT: [[X:%.*]] = load i32, ptr [[GEP]], align 4
+; SCALABLE-NEXT: [[Y:%.*]] = add i32 [[X]], 1
+; SCALABLE-NEXT: store i32 [[Y]], ptr [[GEP]], align 4
+; SCALABLE-NEXT: store i32 [[Y]], ptr [[Q]], align 4
+; SCALABLE-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; SCALABLE-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; SCALABLE-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
+; SCALABLE: [[EXIT]]:
+; SCALABLE-NEXT: ret void
+;
+; FIXEDLEN-LABEL: define void @uniform_load_store(
+; FIXEDLEN-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; FIXEDLEN-NEXT: [[ENTRY:.*]]:
+; FIXEDLEN-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 17
+; FIXEDLEN-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; FIXEDLEN: [[VECTOR_SCEVCHECK]]:
+; FIXEDLEN-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; FIXEDLEN-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
+; FIXEDLEN-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; FIXEDLEN: [[VECTOR_MEMCHECK]]:
+; FIXEDLEN-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[Q]], i64 4
+; FIXEDLEN-NEXT: [[TMP2:%.*]] = add i32 [[N]], -1
+; FIXEDLEN-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; FIXEDLEN-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; FIXEDLEN-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 4
+; FIXEDLEN-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; FIXEDLEN-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP1]]
+; FIXEDLEN-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[P]], [[SCEVGEP]]
+; FIXEDLEN-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; FIXEDLEN-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; FIXEDLEN: [[VECTOR_PH]]:
+; FIXEDLEN-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
+; FIXEDLEN-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
+; FIXEDLEN-NEXT: br label %[[VECTOR_BODY:.*]]
+; FIXEDLEN: [[VECTOR_BODY]]:
+; FIXEDLEN-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FIXEDLEN-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; FIXEDLEN-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i64 8
+; FIXEDLEN-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP6]], align 4, !alias.scope [[META18:![0-9]+]]
+; FIXEDLEN-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i32>, ptr [[TMP7]], align 4, !alias.scope [[META18]]
+; FIXEDLEN-NEXT: [[TMP8:%.*]] = add <8 x i32> [[WIDE_LOAD]], splat (i32 1)
+; FIXEDLEN-NEXT: [[TMP9:%.*]] = add <8 x i32> [[WIDE_LOAD2]], splat (i32 1)
+; FIXEDLEN-NEXT: store <8 x i32> [[TMP8]], ptr [[TMP6]], align 4, !alias.scope [[META18]]
+; FIXEDLEN-NEXT: store <8 x i32> [[TMP9]], ptr [[TMP7]], align 4, !alias.scope [[META18]]
+; FIXEDLEN-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP9]], i32 7
+; FIXEDLEN-NEXT: store i32 [[TMP10]], ptr [[Q]], align 4, !alias.scope [[META21:![0-9]+]], !noalias [[META18]]
+; FIXEDLEN-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; FIXEDLEN-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; FIXEDLEN-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; FIXEDLEN: [[MIDDLE_BLOCK]]:
+; FIXEDLEN-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; FIXEDLEN-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; FIXEDLEN: [[SCALAR_PH]]:
+; FIXEDLEN-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; FIXEDLEN-NEXT: br label %[[LOOP:.*]]
+; FIXEDLEN: [[LOOP]]:
+; FIXEDLEN-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; FIXEDLEN-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
+; FIXEDLEN-NEXT: [[X:%.*]] = load i32, ptr [[GEP]], align 4
+; FIXEDLEN-NEXT: [[Y:%.*]] = add i32 [[X]], 1
+; FIXEDLEN-NEXT: store i32 [[Y]], ptr [[GEP]], align 4
+; FIXEDLEN-NEXT: store i32 [[Y]], ptr [[Q]], align 4
+; FIXEDLEN-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; FIXEDLEN-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; FIXEDLEN-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
+; FIXEDLEN: [[EXIT]]:
+; FIXEDLEN-NEXT: ret void
+;
+; TF-SCALABLE-LABEL: define void @uniform_load_store(
+; TF-SCALABLE-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; TF-SCALABLE-NEXT: [[ENTRY:.*:]]
+; TF-SCALABLE-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; TF-SCALABLE: [[VECTOR_SCEVCHECK]]:
+; TF-SCALABLE-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; TF-SCALABLE-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
+; TF-SCALABLE-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; TF-SCALABLE: [[VECTOR_MEMCHECK]]:
+; TF-SCALABLE-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[Q]], i64 4
+; TF-SCALABLE-NEXT: [[TMP2:%.*]] = add i32 [[N]], -1
+; TF-SCALABLE-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; TF-SCALABLE-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; TF-SCALABLE-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 4
+; TF-SCALABLE-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; TF-SCALABLE-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP1]]
+; TF-SCALABLE-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[P]], [[SCEVGEP]]
+; TF-SCALABLE-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; TF-SCALABLE-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; TF-SCALABLE: [[VECTOR_PH]]:
+; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[Q]], i64 0
+; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; TF-SCALABLE: [[VECTOR_BODY]]:
+; TF-SCALABLE-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[EVL_BASED_IV]]
+; TF-SCALABLE-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META10:![0-9]+]]
+; TF-SCALABLE-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 1)
+; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META10]]
+; TF-SCALABLE-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP8]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META13:![0-9]+]], !noalias [[META10]]
+; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add i32 [[TMP6]], [[EVL_BASED_IV]]
+; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
+; TF-SCALABLE-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; TF-SCALABLE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; TF-SCALABLE: [[MIDDLE_BLOCK]]:
+; TF-SCALABLE-NEXT: br label %[[EXIT:.*]]
+; TF-SCALABLE: [[SCALAR_PH]]:
+; TF-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; TF-SCALABLE-NEXT: br label %[[LOOP:.*]]
+; TF-SCALABLE: [[LOOP]]:
+; TF-SCALABLE-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; TF-SCALABLE-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
+; TF-SCALABLE-NEXT: [[X:%.*]] = load i32, ptr [[GEP]], align 4
+; TF-SCALABLE-NEXT: [[Y:%.*]] = add i32 [[X]], 1
+; TF-SCALABLE-NEXT: store i32 [[Y]], ptr [[GEP]], align 4
+; TF-SCALABLE-NEXT: store i32 [[Y]], ptr [[Q]], align 4
+; TF-SCALABLE-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; TF-SCALABLE-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; TF-SCALABLE-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
+; TF-SCALABLE: [[EXIT]]:
+; TF-SCALABLE-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep
+ %y = add i32 %x, 1
+ store i32 %y, ptr %gep
+ store i32 %y, ptr %q
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.width", i32 4}
!2 = !{!"llvm.loop.vectorize.enable"}
>From 63fd4c3869058b93b54fc6437691445b8700ee46 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Wed, 17 Dec 2025 21:35:58 -0800
Subject: [PATCH 2/6] [LV] Convert scatter w/uniform addr and mask being header
mask to scalar store.
This patch converts scatter with uniform address and the mask being the
header mask to the `last-active-lane` + `extract-lane` + `scalar store`.
The header mask can guarantee that the scatter has at least one
active lane and can be converted to the scalar store.
Note that some dead instructions will be generated by
`extract-lane`. Will open another PR to clean up.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 9 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 86 ++++---
.../Transforms/Vectorize/VPlanTransforms.cpp | 65 ++++++
.../Transforms/Vectorize/VPlanTransforms.h | 5 +
.../LoopVectorize/AArch64/masked-call.ll | 151 ++++++++----
.../LoopVectorize/AArch64/sve-tail-folding.ll | 8 +-
.../Transforms/LoopVectorize/RISCV/cse.ll | 13 +-
.../Transforms/LoopVectorize/RISCV/divrem.ll | 37 +--
.../RISCV/gather-scatter-cost.ll | 85 +++----
.../LoopVectorize/RISCV/induction-costs.ll | 16 +-
.../RISCV/pointer-induction-rv32.ll | 6 +-
.../LoopVectorize/RISCV/pointer-induction.ll | 21 +-
.../Transforms/LoopVectorize/RISCV/pr88802.ll | 41 ++--
.../RISCV/strided-access-wide-stride.ll | 14 +-
.../RISCV/strided-accesses-narrow-iv.ll | 52 +++--
.../LoopVectorize/RISCV/strided-accesses.ll | 13 +-
.../truncate-to-minimal-bitwidth-cost.ll | 37 ++-
.../truncate-to-minimal-bitwidth-evl-crash.ll | 16 +-
.../RISCV/type-info-cache-evl-crash.ll | 18 +-
.../LoopVectorize/RISCV/uniform-load-store.ll | 217 ++----------------
.../vf-will-not-generate-any-vector-insts.ll | 12 +-
22 files changed, 462 insertions(+), 468 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3c65eda187a6e..911a4e621a234 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6641,8 +6641,6 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
// Now optimize the initial VPlan.
RUN_VPLAN_PASS(VPlanTransforms::hoistPredicatedLoads, *Plan, PSE, OrigLoop);
RUN_VPLAN_PASS(VPlanTransforms::sinkPredicatedStores, *Plan, PSE, OrigLoop);
- RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
- Config.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
if (CM.foldTailWithEVL()) {
@@ -6893,6 +6891,12 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
if (Range.Start.isScalar())
Range.End = Range.Start * 2;
+ RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
+ Config.getMinimalBitwidths());
+ RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *Plan);
+ RUN_VPLAN_PASS(VPlanTransforms::narrowScatters, *Plan, CostCtx, Range,
+ CM.foldTailWithEVL());
+
for (ElementCount VF : Range)
Plan->addVF(VF);
Plan->setName("Initial VPlan");
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 814b77a96e825..1b8b6bedf6b77 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1524,6 +1524,15 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Set the symbolic name for the VPInstruction.
void setName(StringRef NewName) { Name = NewName.str(); }
+ /// Compute the cost for a recipe with \p VF using \p Opcode, \p ValTy.
+ /// This function may not be as accurate as
+ /// `getCostForRecipeWithOpcode` since it only provides type-based queries.
+ static InstructionCost getCostForRecipeWithOpcodeAndTypes(unsigned Opcode,
+ Type *ValTy,
+ Instruction *I,
+ ElementCount VF,
+ VPCostContext &Ctx);
+
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the VPInstruction to \p O.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a5343caccc992..b8118ed558f87 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1114,6 +1114,57 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
}
+InstructionCost VPInstruction::getCostForRecipeWithOpcodeAndTypes(
+ unsigned Opcode, Type *ValTy, Instruction *I, ElementCount VF,
+ VPCostContext &Ctx) {
+ switch (Opcode) {
+ case VPInstruction::LastActiveLane: {
+ // LastActiveLane computes the index of the last active lane in a
+ // predicate mask: NOT + cttz_elts + SUB.
+ if (VF.isScalar())
+ return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ValTy,
+ CmpInst::makeCmpResultType(ValTy),
+ CmpInst::ICMP_EQ, Ctx.CostKind);
+ auto *PredTy = toVectorTy(ValTy, VF);
+ IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts,
+ Type::getInt64Ty(Ctx.LLVMCtx),
+ {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
+ InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ // Add cost of NOT operation on the predicate.
+ Cost += Ctx.TTI.getArithmeticInstrCost(
+ Instruction::Xor, PredTy, Ctx.CostKind,
+ {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+ {TargetTransformInfo::OK_UniformConstantValue,
+ TargetTransformInfo::OP_None});
+ // Add cost of SUB operation on the index.
+ Cost += Ctx.TTI.getArithmeticInstrCost(
+ Instruction::Sub, Type::getInt64Ty(Ctx.LLVMCtx), Ctx.CostKind);
+ return Cost;
+ }
+ case VPInstruction::ExtractLane: {
+ // Compute the cost of ExtractLane for a vector with the given scalar
+ // element type. ExtractLane extracts an element at a runtime-determined
+ // index.
+ if (VF.isScalar())
+ return 0;
+ return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement,
+ toVectorTy(ValTy, VF), Ctx.CostKind);
+ }
+ case Instruction::Store: {
+ assert(I && "Querying cost for store must have underlying instruction.");
+ return Ctx.TTI.getMemoryOpCost(Opcode, ValTy, getLoadStoreAlignment(I),
+ getLoadStoreAddressSpace(I), Ctx.CostKind,
+ TTI::getOperandInfo(I->getOperand(0)), I);
+ }
+ case Instruction::Sub: {
+ return Ctx.TTI.getArithmeticInstrCost(Opcode, toVectorTy(ValTy, VF),
+ Ctx.CostKind);
+ }
+ default:
+ llvm_unreachable("Unsupported opcode");
+ }
+}
+
InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
unsigned Opcode, ElementCount VF, VPCostContext &Ctx) const {
Type *ScalarTy = this->getScalarType();
@@ -1349,16 +1400,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
}
case Instruction::ExtractElement:
case VPInstruction::ExtractLane: {
- if (VF.isScalar()) {
- // ExtractLane with VF=1 takes care of handling extracting across multiple
- // parts.
- return 0;
- }
-
- // Add on the cost of extracting the element.
- auto *VecTy = toVectorTy(getOperand(0)->getScalarType(), VF);
- return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
- Ctx.CostKind);
+ return getCostForRecipeWithOpcodeAndTypes(
+ getOpcode(), getOperand(0)->getScalarType(), nullptr, VF, Ctx);
}
case VPInstruction::AnyOf: {
auto *VecTy = toVectorTy(this->getScalarType(), VF);
@@ -1379,26 +1422,9 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
}
case VPInstruction::LastActiveLane: {
- Type *Ty = this->getScalarType();
- Type *ScalarTy = getOperand(0)->getScalarType();
- if (VF.isScalar())
- return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy,
- CmpInst::makeCmpResultType(ScalarTy),
- CmpInst::ICMP_EQ, Ctx.CostKind);
- // Calculate the cost of determining the lane index: NOT + cttz_elts + SUB.
- auto *PredTy = toVectorTy(ScalarTy, VF);
- IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts, Ty,
- {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
- InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
- // Add cost of NOT operation on the predicate.
- Cost += Ctx.TTI.getArithmeticInstrCost(
- Instruction::Xor, PredTy, Ctx.CostKind,
- {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
- {TargetTransformInfo::OK_UniformConstantValue,
- TargetTransformInfo::OP_None});
- // Add cost of SUB operation on the index.
- Cost += Ctx.TTI.getArithmeticInstrCost(Instruction::Sub, Ty, Ctx.CostKind);
- return Cost;
+ return getCostForRecipeWithOpcodeAndTypes(VPInstruction::LastActiveLane,
+ getOperand(0)->getScalarType(),
+ nullptr, VF, Ctx);
}
case VPInstruction::ExtractLastActive: {
Type *ScalarTy = this->getScalarType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5f125aecb005a..9f8a1cc149713 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1733,6 +1733,71 @@ getUnmaskedDivRemOpcode(Intrinsic::ID ID) {
}
}
+void VPlanTransforms::narrowScatters(VPlan &Plan, VPCostContext &Ctx,
+ VFRange &Range, bool FoldTailWithEVL) {
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ // Convert an unmasked or header masked scatter with a uniform address
+ // into extract-last-lane + scalar store.
+ auto *WidenStoreR = dyn_cast<VPWidenStoreRecipe>(&R);
+ if (!WidenStoreR ||
+ !vputils::isUniformAcrossVFsAndUFs(WidenStoreR->getAddr()) ||
+ WidenStoreR->isConsecutive())
+ continue;
+
+ // Convert the scatter to a scalar store if it is header masked.
+ VPValue *Mask = WidenStoreR->getMask();
+ if (!Mask || !match(Mask, m_HeaderMask()))
+ continue;
+
+ // If the body is header-masked, it guarantees each iteration has at
+ // least one active lane. So it is safe to convert the scatter to a
+ // scalar store.
+ if (!LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) {
+ InstructionCost ScatterCost = WidenStoreR->computeCost(VF, Ctx);
+ Type *ValTy = WidenStoreR->getStoredValue()->getScalarType();
+
+ // ScalarCost = LastActiveLaneCost + ExtractLaneCost +
+ // ScalarStoreCost.
+ InstructionCost ScalarCost = 0;
+
+ // LastActiveLane can lower to EVL - 1 when folding tail with
+ // EVL.
+ ScalarCost +=
+ FoldTailWithEVL
+ ? VPInstruction::getCostForRecipeWithOpcodeAndTypes(
+ Instruction::Sub, Type::getInt32Ty(Ctx.LLVMCtx),
+ nullptr, ElementCount::getFixed(1), Ctx)
+ : VPInstruction::getCostForRecipeWithOpcodeAndTypes(
+ VPInstruction::LastActiveLane,
+ Type::getInt1Ty(Ctx.LLVMCtx), nullptr, VF, Ctx);
+ ScalarCost += VPInstruction::getCostForRecipeWithOpcodeAndTypes(
+ VPInstruction::ExtractLane, ValTy, nullptr, VF, Ctx);
+ ScalarCost += VPInstruction::getCostForRecipeWithOpcodeAndTypes(
+ Instruction::Store, ValTy, &WidenStoreR->getIngredient(),
+ VF, Ctx);
+
+ return ScalarCost <= ScatterCost;
+ },
+ Range))
+ continue;
+ VPBuilder Builder(WidenStoreR);
+ VPInstruction *LastActiveLane = Builder.createLastActiveLane(Mask);
+ VPInstruction *Extract =
+ Builder.createNaryOp(VPInstruction::ExtractLane,
+ {LastActiveLane, WidenStoreR->getStoredValue()});
+ auto *ScalarStore = new VPReplicateRecipe(
+ &WidenStoreR->getIngredient(), {Extract, WidenStoreR->getAddr()},
+ /*IsSingleScalar*/ true, /*Mask*/ nullptr, {},
+ /*Metadata*/ *WidenStoreR);
+ ScalarStore->insertBefore(WidenStoreR);
+ WidenStoreR->eraseFromParent();
+ }
+ }
+}
+
static void narrowToSingleScalarRecipes(VPlan &Plan) {
if (Plan.hasScalarVFOnly())
return;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index af8f6de26e0f9..6a74fa9577d08 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -599,6 +599,11 @@ struct VPlanTransforms {
static void makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
VPCostContext &CostCtx);
+
+ /// Convert the scatter to extract-last-active-lane + scalar store if
+ /// profitable.
+ static void narrowScatters(VPlan &Plan, VPCostContext &Ctx, VFRange &Range,
+ bool FoldTailWithEVL);
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index cb2eefcbc9b91..fdf561485a19b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -964,26 +964,73 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFNONE: [[END]]:
; TFNONE-NEXT: ret void
;
-; TFCOMMON-LABEL: define void @test_widen_exp_v2(
-; TFCOMMON-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
-; TFCOMMON-NEXT: [[ENTRY:.*]]:
-; TFCOMMON-NEXT: br label %[[LOOP:.*]]
-; TFCOMMON: [[LOOP]]:
-; TFCOMMON-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_END:.*]] ]
-; TFCOMMON-NEXT: [[LD:%.*]] = load double, ptr [[P2]], align 8
-; TFCOMMON-NEXT: [[EXP:%.*]] = tail call double @llvm.exp.f64(double [[LD]]) #[[ATTR7:[0-9]+]]
-; TFCOMMON-NEXT: [[COND1:%.*]] = fcmp ogt double [[EXP]], 0.000000e+00
-; TFCOMMON-NEXT: br i1 [[COND1]], label %[[LOOP_MIDDLE:.*]], label %[[LOOP_END]]
-; TFCOMMON: [[LOOP_MIDDLE]]:
-; TFCOMMON-NEXT: br label %[[LOOP_END]]
-; TFCOMMON: [[LOOP_END]]:
-; TFCOMMON-NEXT: [[SINK:%.*]] = phi double [ 0.000000e+00, %[[LOOP_MIDDLE]] ], [ 1.000000e+00, %[[LOOP]] ]
-; TFCOMMON-NEXT: store double [[SINK]], ptr [[P]], align 8
-; TFCOMMON-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; TFCOMMON-NEXT: [[COND2:%.*]] = icmp eq i64 [[IV]], [[N]]
-; TFCOMMON-NEXT: br i1 [[COND2]], label %[[END:.*]], label %[[LOOP]]
-; TFCOMMON: [[END]]:
-; TFCOMMON-NEXT: ret void
+; TFALWAYS-LABEL: define void @test_widen_exp_v2(
+; TFALWAYS-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
+; TFALWAYS-NEXT: [[ENTRY:.*:]]
+; TFALWAYS-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; TFALWAYS-NEXT: br label %[[VECTOR_PH:.*]]
+; TFALWAYS: [[VECTOR_PH]]:
+; TFALWAYS-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; TFALWAYS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
+; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[TMP0]])
+; TFALWAYS-NEXT: br label %[[VECTOR_BODY:.*]]
+; TFALWAYS: [[VECTOR_BODY]]:
+; TFALWAYS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFALWAYS-NEXT: [[TMP3:%.*]] = load double, ptr [[P2]], align 8
+; TFALWAYS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[TMP3]], i64 0
+; TFALWAYS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+; TFALWAYS-NEXT: [[TMP4:%.*]] = call <vscale x 2 x double> @exp_masked_scalable(<vscale x 2 x double> [[BROADCAST_SPLAT]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFALWAYS-NEXT: [[TMP5:%.*]] = fcmp ogt <vscale x 2 x double> [[TMP4]], zeroinitializer
+; TFALWAYS-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP5]], <vscale x 2 x double> zeroinitializer, <vscale x 2 x double> splat (double 1.000000e+00)
+; TFALWAYS-NEXT: [[TMP6:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; TFALWAYS-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP6]], i1 false)
+; TFALWAYS-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; TFALWAYS-NEXT: [[TMP7:%.*]] = extractelement <vscale x 2 x double> [[PREDPHI]], i64 [[LAST_ACTIVE_LANE]]
+; TFALWAYS-NEXT: store double [[TMP7]], ptr [[P]], align 8
+; TFALWAYS-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; TFALWAYS-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFALWAYS-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
+; TFALWAYS-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; TFALWAYS: [[MIDDLE_BLOCK]]:
+; TFALWAYS-NEXT: br label %[[END:.*]]
+; TFALWAYS: [[END]]:
+; TFALWAYS-NEXT: ret void
+;
+; TFFALLBACK-LABEL: define void @test_widen_exp_v2(
+; TFFALLBACK-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
+; TFFALLBACK-NEXT: [[ENTRY:.*:]]
+; TFFALLBACK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; TFFALLBACK-NEXT: br label %[[VECTOR_PH:.*]]
+; TFFALLBACK: [[VECTOR_PH]]:
+; TFFALLBACK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; TFFALLBACK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
+; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[TMP0]])
+; TFFALLBACK-NEXT: br label %[[VECTOR_BODY:.*]]
+; TFFALLBACK: [[VECTOR_BODY]]:
+; TFFALLBACK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFFALLBACK-NEXT: [[TMP3:%.*]] = load double, ptr [[P2]], align 8
+; TFFALLBACK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[TMP3]], i64 0
+; TFFALLBACK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+; TFFALLBACK-NEXT: [[TMP4:%.*]] = call <vscale x 2 x double> @exp_masked_scalable(<vscale x 2 x double> [[BROADCAST_SPLAT]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFFALLBACK-NEXT: [[TMP5:%.*]] = fcmp ogt <vscale x 2 x double> [[TMP4]], zeroinitializer
+; TFFALLBACK-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP5]], <vscale x 2 x double> zeroinitializer, <vscale x 2 x double> splat (double 1.000000e+00)
+; TFFALLBACK-NEXT: [[TMP6:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; TFFALLBACK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP6]], i1 false)
+; TFFALLBACK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; TFFALLBACK-NEXT: [[TMP7:%.*]] = extractelement <vscale x 2 x double> [[PREDPHI]], i64 [[LAST_ACTIVE_LANE]]
+; TFFALLBACK-NEXT: store double [[TMP7]], ptr [[P]], align 8
+; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; TFFALLBACK-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFFALLBACK-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
+; TFFALLBACK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; TFFALLBACK: [[MIDDLE_BLOCK]]:
+; TFFALLBACK-NEXT: br label %[[END:.*]]
+; TFFALLBACK: [[END]]:
+; TFFALLBACK-NEXT: ret void
;
; TFA_INTERLEAVE-LABEL: define void @test_widen_exp_v2(
; TFA_INTERLEAVE-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
@@ -991,31 +1038,53 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_PH:.*]]
; TFA_INTERLEAVE: [[VECTOR_PH]]:
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[TMP0]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[TMP0]]
+; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP4]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP2]], i64 [[TMP0]])
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
-; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE4:.*]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE4]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[PRED_STORE_CONTINUE4]] ]
+; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT1:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
-; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR7:[0-9]+]]
-; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = fcmp ogt double [[TMP2]], 0.000000e+00
-; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP3]], double 0.000000e+00, double 1.000000e+00
-; TFA_INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; TFA_INTERLEAVE: [[PRED_STORE_IF]]:
-; TFA_INTERLEAVE-NEXT: store double [[PREDPHI]], ptr [[P]], align 8
-; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE]]:
-; TFA_INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4]]
-; TFA_INTERLEAVE: [[PRED_STORE_IF3]]:
+; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[TMP1]], i64 0
+; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP27:%.*]] = call <vscale x 2 x double> @exp_masked_scalable(<vscale x 2 x double> [[BROADCAST_SPLAT]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFA_INTERLEAVE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x double> @exp_masked_scalable(<vscale x 2 x double> [[BROADCAST_SPLAT]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = fcmp ogt <vscale x 2 x double> [[TMP27]], zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP8:%.*]] = fcmp ogt <vscale x 2 x double> [[TMP6]], zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[PREDPHI1:%.*]] = select <vscale x 2 x i1> [[TMP7]], <vscale x 2 x double> zeroinitializer, <vscale x 2 x double> splat (double 1.000000e+00)
+; TFA_INTERLEAVE-NEXT: [[PREDPHI3:%.*]] = select <vscale x 2 x i1> [[TMP8]], <vscale x 2 x double> zeroinitializer, <vscale x 2 x double> splat (double 1.000000e+00)
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], splat (i1 true)
+; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
+; TFA_INTERLEAVE-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP10]], i1 false)
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = add i64 [[TMP13]], [[FIRST_INACTIVE_LANE]]
+; TFA_INTERLEAVE-NEXT: [[FIRST_INACTIVE_LANE4:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP9]], i1 false)
+; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = mul i64 [[TMP12]], 0
+; TFA_INTERLEAVE-NEXT: [[TMP16:%.*]] = add i64 [[TMP15]], [[FIRST_INACTIVE_LANE4]]
+; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE4]], [[TMP12]]
+; TFA_INTERLEAVE-NEXT: [[TMP18:%.*]] = select i1 [[TMP17]], i64 [[TMP16]], i64 [[TMP14]]
+; TFA_INTERLEAVE-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[TMP18]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 2
+; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 0
+; TFA_INTERLEAVE-NEXT: [[TMP22:%.*]] = extractelement <vscale x 2 x double> [[PREDPHI1]], i64 [[LAST_ACTIVE_LANE]]
+; TFA_INTERLEAVE-NEXT: [[TMP23:%.*]] = mul i64 [[TMP20]], 1
+; TFA_INTERLEAVE-NEXT: [[TMP24:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], [[TMP23]]
+; TFA_INTERLEAVE-NEXT: [[TMP25:%.*]] = extractelement <vscale x 2 x double> [[PREDPHI3]], i64 [[TMP24]]
+; TFA_INTERLEAVE-NEXT: [[TMP26:%.*]] = icmp uge i64 [[LAST_ACTIVE_LANE]], [[TMP23]]
+; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP26]], double [[TMP25]], double [[TMP22]]
; TFA_INTERLEAVE-NEXT: store double [[PREDPHI]], ptr [[P]], align 8
-; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE4]]
-; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE4]]:
-; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX_NEXT]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[TMP0]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP4]], [[TMP0]]
+; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
+; TFA_INTERLEAVE-NEXT: [[TMP28:%.*]] = add i64 [[INDEX_NEXT]], [[TMP2]]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP28]], i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; TFA_INTERLEAVE: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll
index 27b5cdc4001bb..c7344b3a8f7ea 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll
@@ -358,15 +358,17 @@ define void @uniform_store(ptr noalias %dst, ptr noalias readonly %src, i64 %n)
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N:%.*]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[TMP3:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP3]], i1 false)
+; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-NEXT: store i32 [[TMP4]], ptr [[DST:%.*]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/cse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/cse.ll
index be59bc1769eac..bf8e9707a4326 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/cse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/cse.ll
@@ -10,10 +10,6 @@ define i32 @widenpointerinduction_evl_cse(ptr noalias %p0, ptr noalias %p1) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[P0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[P1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P0]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
@@ -22,10 +18,13 @@ define i32 @widenpointerinduction_evl_cse(ptr noalias %p0, ptr noalias %p1) {
; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
; CHECK-NEXT: [[TMP1:%.*]] = shl <vscale x 4 x i32> [[TMP0]], splat (i32 1)
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI3]], <vscale x 4 x i32> [[TMP1]]
-; CHECK-NEXT: [[VECTOR_GEP4:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i32> [[TMP1]]
+; CHECK-NEXT: [[VECTOR_GEP2:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i32> [[TMP1]]
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4p0.nxv4p0(<vscale x 4 x ptr> [[VECTOR_GEP4]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4p0.nxv4p0(<vscale x 4 x ptr> [[VECTOR_GEP]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
+; CHECK-NEXT: [[TMP6:%.*]] = sub i32 [[TMP2]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <vscale x 4 x ptr> [[VECTOR_GEP2]], i32 [[TMP6]]
+; CHECK-NEXT: store ptr [[TMP7]], ptr [[P0]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x ptr> [[VECTOR_GEP]], i32 [[TMP6]]
+; CHECK-NEXT: store ptr [[TMP5]], ptr [[P1]], align 4
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
; CHECK-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 1
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 [[TMP3]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index d02dc36f7a6ae..6536d5372bf41 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -631,7 +631,7 @@ for.end:
define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-LABEL: @udiv_sdiv_with_invariant_divisors(
; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
@@ -640,30 +640,31 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[Y:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[P:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.stepvector.nxv4i8()
; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i8> splat (i8 -12), [[TMP1]]
-; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
+; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[INDUCTION]], [[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP_LATCH]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 12, [[VECTOR_BODY]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[INDUCTION]], [[LOOP_LATCH]] ], [ [[VEC_IND_NEXT:%.*]], [[EXIT]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 12, [[LOOP_LATCH]] ], [ [[AVL_NEXT:%.*]], [[EXIT]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i8> @llvm.vp.udiv.nxv4i8(<vscale x 4 x i8> [[VEC_IND]], <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
-; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[TMP5]] to <vscale x 4 x i16>
-; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vp.sdiv.nxv4i16(<vscale x 4 x i16> [[TMP6]], <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
-; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i16> [[TMP8]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP9]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i8> @llvm.vp.udiv.nxv4i8(<vscale x 4 x i8> [[VEC_IND]], <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
+; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 4 x i8> [[TMP4]] to <vscale x 4 x i16>
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i16> @llvm.vp.sdiv.nxv4i16(<vscale x 4 x i16> [[TMP5]], <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
+; CHECK-NEXT: [[TMP7:%.*]] = sext <vscale x 4 x i16> [[TMP6]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 1
+; CHECK-NEXT: [[MERGE:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI]], i64 [[TMP9]]
+; CHECK-NEXT: store i32 [[MERGE]], ptr [[P:%.*]], align 4
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT8]]
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP_LATCH]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT6]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: br label [[EXIT:%.*]]
+; CHECK-NEXT: br label [[EXIT1:%.*]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
index 77805d7fd8f51..75b388187a194 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
@@ -5,14 +5,14 @@
define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK-LABEL: @predicated_uniform_load(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[IBOX:%.*]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[N:%.*]] to i64
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1
; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 0)
; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[SMAX2]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add nuw i32 [[TMP2]], 1
; CHECK-NEXT: br label [[VECTOR_SCEVCHECK:%.*]]
; CHECK: vector.scevcheck:
-; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[IBOX]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[N]] to i64
; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 1
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP5]], i64 0)
; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[SMAX]] to i32
@@ -21,30 +21,31 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP9]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[NBRBOXES:%.*]], i64 4
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[BOXES:%.*]], i64 4
-; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[NBRBOXES]], [[SCEVGEP1]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[BOXES]], [[SCEVGEP]]
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 4
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 4
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[COND:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT1]], splat (i1 true)
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[BOXES]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[NBRBOXES]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[COND:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[SRC]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP13]], i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP10]], i32 [[TMP11]]), !alias.scope [[META0:![0-9]+]]
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[COND]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP10]]
-; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT: [[TMP13:%.*]] = sub i64 [[TMP12]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI]], i64 [[TMP13]]
+; CHECK-NEXT: store i32 [[TMP14]], ptr [[DST]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP11]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: scalar.ph:
@@ -55,13 +56,13 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK: loop.then:
; CHECK-NEXT: br label [[LOOP_LATCH]]
; CHECK: loop.else:
-; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[BOXES]], align 4
+; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[SRC]], align 4
; CHECK-NEXT: br label [[LOOP_LATCH]]
; CHECK: loop.latch:
-; CHECK-NEXT: [[STORE:%.*]] = phi i32 [ [[TMP17]], [[LOOP_ELSE]] ], [ 0, [[LOOP_THEN]] ]
-; CHECK-NEXT: store i32 [[STORE]], ptr [[NBRBOXES]], align 4
+; CHECK-NEXT: [[STORE:%.*]] = phi i32 [ [[TMP16]], [[LOOP_ELSE]] ], [ 0, [[LOOP_THEN]] ]
+; CHECK-NEXT: store i32 [[STORE]], ptr [[DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp sgt i32 [[IV]], [[IBOX]]
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp sgt i32 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: ret void
@@ -139,32 +140,32 @@ define void @store_to_addr_generated_from_invariant_addr(ptr noalias %p0, ptr no
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[P0:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[P0:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP4]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[P1:%.*]], <vscale x 2 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2p0.nxv2p0(<vscale x 2 x ptr> [[BROADCAST_SPLAT1]], <vscale x 2 x ptr> align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: [[TMP6:%.*]] = load i64, ptr [[P2:%.*]], align 4
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P3:%.*]], i64 [[TMP6]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> align 4 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> align 4 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> zeroinitializer, <vscale x 2 x ptr> align 1 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P1:%.*]], <vscale x 2 x i64> [[VEC_IND]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2p0.nxv2p0(<vscale x 2 x ptr> [[BROADCAST_SPLAT]], <vscale x 2 x ptr> align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
+; CHECK-NEXT: [[TMP5:%.*]] = load i64, ptr [[P2:%.*]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P3:%.*]], i64 [[TMP5]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> zeroinitializer, <vscale x 2 x ptr> align 1 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
index 1752ea0814fcc..f7686fadedc32 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
@@ -126,28 +126,28 @@ define void @test_3_inductions(ptr noalias %dst, ptr noalias %src, i64 %n) #1 {
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
; CHECK-NEXT: [[TMP2:%.*]] = mul <vscale x 2 x i32> [[TMP1]], splat (i32 2)
; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 2 x i32> splat (i32 1), [[TMP2]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <vscale x 2 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <vscale x 2 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[TMP4:%.*]] = shl i32 [[TMP3]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP4]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = or <vscale x 2 x i32> [[VEC_IND1]], [[VEC_IND]]
; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 2 x i32> [[TMP5]] to <vscale x 2 x i64>
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], <vscale x 2 x i64> [[TMP6]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2p0.nxv2p0(<vscale x 2 x ptr> [[TMP7]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP8]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x ptr> [[TMP7]], i64 [[TMP11]]
+; CHECK-NEXT: store ptr [[TMP10]], ptr [[DST]], align 8
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT3]]
-; CHECK-NEXT: [[VEC_IND_NEXT4]] = add <vscale x 2 x i32> [[VEC_IND1]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT2]] = add <vscale x 2 x i32> [[VEC_IND1]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction-rv32.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction-rv32.ll
index 97bf31d77e00e..e5153ab3673b0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction-rv32.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction-rv32.ll
@@ -9,8 +9,6 @@ define i32 @widenpointerinduction_evl(ptr noalias %p) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[P]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
@@ -18,7 +16,9 @@ define i32 @widenpointerinduction_evl(ptr noalias %p) {
; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i32> [[TMP0]]
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4p0.nxv4p0(<vscale x 4 x ptr> [[VECTOR_GEP]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[TMP1]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x ptr> [[VECTOR_GEP]], i32 [[TMP4]]
+; CHECK-NEXT: store ptr [[TMP3]], ptr [[P]], align 4
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index a55985caa28ab..bf29404fc6b68 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -13,10 +13,6 @@ define void @ptr_induction(ptr %p, ptr noalias %q, ptr noalias %p.end) #0 {
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP8]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[Q]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[P]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -26,13 +22,16 @@ define void @ptr_induction(ptr %p, ptr noalias %q, ptr noalias %p.end) #0 {
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint <vscale x 2 x ptr> [[VECTOR_GEP]] to <vscale x 2 x i64>
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP6]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP4]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP4]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = ptrtoint <vscale x 2 x ptr> [[VECTOR_GEP]] to <vscale x 2 x i64>
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i64> [[TMP9]], i64 [[TMP6]]
+; CHECK-NEXT: store i64 [[TMP10]], ptr [[Q]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i64> [[VEC_IND]], i64 [[TMP6]]
+; CHECK-NEXT: store i64 [[TMP8]], ptr [[P]], align 8
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT6]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[PTR_IND7]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -74,8 +73,6 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[START]], i64 4
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[DST]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[END]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT6]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_MEMCHECK]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
index 140ce955d4b49..10ca2c2ae83da 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
@@ -7,31 +7,32 @@ define void @test(ptr %p, i64 %a, i8 %b) {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i8> poison, i8 [[B]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[B]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[A]], 48
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = ashr <vscale x 2 x i64> [[BROADCAST_SPLAT2]], splat (i64 52)
-; CHECK-NEXT: [[TMP7:%.*]] = trunc <vscale x 2 x i64> [[TMP6]] to <vscale x 2 x i32>
-; CHECK-NEXT: [[TMP8:%.*]] = zext <vscale x 2 x i8> [[BROADCAST_SPLAT]] to <vscale x 2 x i32>
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[P]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <vscale x 8 x i64> [[BROADCAST_SPLAT2]], splat (i64 52)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc <vscale x 8 x i64> [[TMP1]] to <vscale x 8 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = zext <vscale x 8 x i8> [[BROADCAST_SPLAT]] to <vscale x 8 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 8 x i32> @llvm.stepvector.nxv8i32()
; CHECK-NEXT: br label [[FOR_COND:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_COND]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_COND]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 9, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[FOR_COND]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP11]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP12:%.*]] = icmp slt <vscale x 2 x i32> [[VEC_IND]], splat (i32 2)
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP12]], <vscale x 2 x i32> [[TMP8]], <vscale x 2 x i32> [[TMP7]]
-; CHECK-NEXT: [[TMP16:%.*]] = shl <vscale x 2 x i32> [[PREDPHI]], splat (i32 8)
-; CHECK-NEXT: [[TMP17:%.*]] = trunc <vscale x 2 x i32> [[TMP16]] to <vscale x 2 x i8>
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> [[TMP17]], <vscale x 2 x ptr> align 1 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[TMP11]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = icmp slt <vscale x 8 x i32> [[VEC_IND]], splat (i32 2)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP6]], <vscale x 8 x i32> [[TMP3]], <vscale x 8 x i32> [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = shl <vscale x 8 x i32> [[PREDPHI]], splat (i32 8)
+; CHECK-NEXT: [[TMP8:%.*]] = trunc <vscale x 8 x i32> [[TMP7]] to <vscale x 8 x i8>
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT: [[TMP10:%.*]] = sub i64 [[TMP9]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 8 x i8> [[TMP8]], i64 [[TMP10]]
+; CHECK-NEXT: store i8 [[TMP12]], ptr [[P]], align 1
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP11]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT8]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_COND]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
index dbee392ed67e9..328fd13f93eed 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
@@ -9,6 +9,9 @@ define void @stride_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst,
; CHECK-LABEL: define void @stride_exceeds_i32_max(
; CHECK-SAME: ptr noalias readonly [[SRC:%.*]], ptr noalias [[DST:%.*]], i16 [[START:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = sext i16 [[START]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = mul nsw i64 [[TMP0]], 3000000000
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i16 -1, [[START]]
; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 1
@@ -26,12 +29,15 @@ define void @stride_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst,
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP4]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT: [[TMP7:%.*]] = sext i32 [[INDEX]] to i64
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP7]], 3000000000
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP11]]
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP8]], i64 3000000000, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP9]], <vscale x 16 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
+; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP8]], i64 3000000000, <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; CHECK-NEXT: [[TMP15:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[TMP15]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 8 x i8> [[TMP14]], i64 [[TMP12]]
+; CHECK-NEXT: store i8 [[TMP13]], ptr [[DST]], align 1
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP6]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll
index 3f85bbc5b8100..abb47efa8b229 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll
@@ -8,8 +8,6 @@ define void @narrow_iv_i8_sext_i64(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[ENTRY:.*:]]
; RV64-NEXT: br label %[[VECTOR_PH:.*]]
; RV64: [[VECTOR_PH]]:
-; RV64-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[OUT]], i64 0
-; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
; RV64-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -19,7 +17,10 @@ define void @narrow_iv_i8_sext_i64(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[TMP5:%.*]] = shl i64 [[TMP1]], 12
; RV64-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP5]]
; RV64-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP2]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; RV64-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
+; RV64-NEXT: [[TMP6:%.*]] = sub i64 [[TMP8]], 1
+; RV64-NEXT: [[TMP7:%.*]] = extractelement <vscale x 8 x i8> [[TMP3]], i64 [[TMP6]]
+; RV64-NEXT: store i8 [[TMP7]], ptr [[OUT]], align 1
; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
; RV64-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
; RV64-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -34,8 +35,6 @@ define void @narrow_iv_i8_sext_i64(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[ENTRY:.*:]]
; RV32-NEXT: br label %[[VECTOR_PH:.*]]
; RV32: [[VECTOR_PH]]:
-; RV32-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[OUT]], i64 0
-; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
; RV32-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -44,7 +43,9 @@ define void @narrow_iv_i8_sext_i64(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 12
; RV32-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i32 [[TMP1]]
; RV32-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[TMP2]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; RV32-NEXT: [[TMP6:%.*]] = sub i32 [[TMP0]], 1
+; RV32-NEXT: [[TMP5:%.*]] = extractelement <vscale x 8 x i8> [[TMP3]], i32 [[TMP6]]
+; RV32-NEXT: store i8 [[TMP5]], ptr [[OUT]], align 1
; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
; RV32-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
; RV32-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -77,8 +78,6 @@ define void @narrow_iv_i8_sext_i16(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[ENTRY:.*:]]
; RV64-NEXT: br label %[[VECTOR_PH:.*]]
; RV64: [[VECTOR_PH]]:
-; RV64-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[OUT]], i64 0
-; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
; RV64-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -88,7 +87,10 @@ define void @narrow_iv_i8_sext_i16(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[TMP5:%.*]] = shl i64 [[TMP1]], 12
; RV64-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP5]]
; RV64-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP2]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; RV64-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
+; RV64-NEXT: [[TMP6:%.*]] = sub i64 [[TMP8]], 1
+; RV64-NEXT: [[TMP7:%.*]] = extractelement <vscale x 8 x i8> [[TMP3]], i64 [[TMP6]]
+; RV64-NEXT: store i8 [[TMP7]], ptr [[OUT]], align 1
; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
; RV64-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
; RV64-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -103,8 +105,6 @@ define void @narrow_iv_i8_sext_i16(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[ENTRY:.*:]]
; RV32-NEXT: br label %[[VECTOR_PH:.*]]
; RV32: [[VECTOR_PH]]:
-; RV32-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[OUT]], i64 0
-; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
; RV32-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -113,7 +113,9 @@ define void @narrow_iv_i8_sext_i16(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 12
; RV32-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i32 [[TMP1]]
; RV32-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[TMP2]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; RV32-NEXT: [[TMP6:%.*]] = sub i32 [[TMP0]], 1
+; RV32-NEXT: [[TMP5:%.*]] = extractelement <vscale x 8 x i8> [[TMP3]], i32 [[TMP6]]
+; RV32-NEXT: store i8 [[TMP5]], ptr [[OUT]], align 1
; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
; RV32-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
; RV32-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -146,8 +148,6 @@ define void @narrow_iv_i8_zext_i64(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[ENTRY:.*:]]
; RV64-NEXT: br label %[[VECTOR_PH:.*]]
; RV64: [[VECTOR_PH]]:
-; RV64-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[OUT]], i64 0
-; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
; RV64-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -157,7 +157,10 @@ define void @narrow_iv_i8_zext_i64(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[TMP5:%.*]] = shl i64 [[TMP1]], 12
; RV64-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP5]]
; RV64-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP2]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; RV64-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
+; RV64-NEXT: [[TMP6:%.*]] = sub i64 [[TMP8]], 1
+; RV64-NEXT: [[TMP7:%.*]] = extractelement <vscale x 8 x i8> [[TMP3]], i64 [[TMP6]]
+; RV64-NEXT: store i8 [[TMP7]], ptr [[OUT]], align 1
; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
; RV64-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
; RV64-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -172,8 +175,6 @@ define void @narrow_iv_i8_zext_i64(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[ENTRY:.*:]]
; RV32-NEXT: br label %[[VECTOR_PH:.*]]
; RV32: [[VECTOR_PH]]:
-; RV32-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[OUT]], i64 0
-; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
; RV32-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -182,7 +183,9 @@ define void @narrow_iv_i8_zext_i64(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 12
; RV32-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i32 [[TMP1]]
; RV32-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[TMP2]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; RV32-NEXT: [[TMP6:%.*]] = sub i32 [[TMP0]], 1
+; RV32-NEXT: [[TMP5:%.*]] = extractelement <vscale x 8 x i8> [[TMP3]], i32 [[TMP6]]
+; RV32-NEXT: store i8 [[TMP5]], ptr [[OUT]], align 1
; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
; RV32-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
; RV32-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -215,8 +218,6 @@ define void @narrow_iv_i8_zext_i16(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[ENTRY:.*:]]
; RV64-NEXT: br label %[[VECTOR_PH:.*]]
; RV64: [[VECTOR_PH]]:
-; RV64-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[OUT]], i64 0
-; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
; RV64-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -226,7 +227,10 @@ define void @narrow_iv_i8_zext_i16(ptr noalias %arr, ptr noalias %out) {
; RV64-NEXT: [[TMP5:%.*]] = shl i64 [[TMP1]], 12
; RV64-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP5]]
; RV64-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP2]], i64 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; RV64-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
+; RV64-NEXT: [[TMP6:%.*]] = sub i64 [[TMP8]], 1
+; RV64-NEXT: [[TMP7:%.*]] = extractelement <vscale x 8 x i8> [[TMP3]], i64 [[TMP6]]
+; RV64-NEXT: store i8 [[TMP7]], ptr [[OUT]], align 1
; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
; RV64-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
; RV64-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -241,8 +245,6 @@ define void @narrow_iv_i8_zext_i16(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[ENTRY:.*:]]
; RV32-NEXT: br label %[[VECTOR_PH:.*]]
; RV32: [[VECTOR_PH]]:
-; RV32-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[OUT]], i64 0
-; RV32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
; RV32-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -251,7 +253,9 @@ define void @narrow_iv_i8_zext_i16(ptr noalias %arr, ptr noalias %out) {
; RV32-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 12
; RV32-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i32 [[TMP1]]
; RV32-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i32(ptr align 1 [[TMP2]], i32 4096, <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
-; RV32-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP3]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; RV32-NEXT: [[TMP6:%.*]] = sub i32 [[TMP0]], 1
+; RV32-NEXT: [[TMP5:%.*]] = extractelement <vscale x 8 x i8> [[TMP3]], i32 [[TMP6]]
+; RV32-NEXT: store i8 [[TMP5]], ptr [[OUT]], align 1
; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP0]], [[INDEX]]
; RV32-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
; RV32-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index d2972748af795..a00f45427bbd2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -1551,8 +1551,6 @@ define void @type_cache_crash(ptr noalias %a, ptr noalias %b, ptr noalias %c, i3
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i64 48
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x double> poison, double [[C2]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1569,7 +1567,10 @@ define void @type_cache_crash(ptr noalias %a, ptr noalias %b, ptr noalias %c, i3
; CHECK-NEXT: [[TMP12:%.*]] = fsub <vscale x 2 x double> [[BROADCAST_SPLAT2]], [[TMP11]]
; CHECK-NEXT: [[TMP13:%.*]] = fmul <vscale x 2 x double> [[BROADCAST_SPLAT6]], [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 2 x double> @llvm.fmuladd.nxv2f64(<vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[BROADCAST_SPLAT8]], <vscale x 2 x double> [[TMP13]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP14]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; CHECK-NEXT: [[TMP18:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT: [[TMP19:%.*]] = sub i64 [[TMP18]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <vscale x 2 x double> [[TMP14]], i64 [[TMP19]]
+; CHECK-NEXT: store double [[TMP17]], ptr [[C]], align 8
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP7]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -2152,8 +2153,6 @@ define void @iv_start_is_addrec(ptr %p, ptr noalias %q) {
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[Q]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -2162,8 +2161,10 @@ define void @iv_start_is_addrec(ptr %p, ptr noalias %q) {
; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[INDEX]], -8
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP4]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP6]], i64 -8, <vscale x 8 x i1> splat (i1 true), i32 [[TMP2]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[WIDE_MASKED_GATHER]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP3]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <vscale x 8 x i8> [[WIDE_MASKED_GATHER]], i64 [[TMP8]]
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[Q]], align 1
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP3]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index 9b82cbf818b99..675cc32a8459d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -108,22 +108,22 @@ exit:
define void @truncate_to_i1_used_by_branch(i8 %x, ptr %dst) #0 {
; CHECK-LABEL: define void @truncate_to_i1_used_by_branch(
; CHECK-SAME: i8 [[X:%.*]], ptr [[DST:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[VECTOR_PH:.*]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 9, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i8.nxv4p0(<vscale x 4 x i8> zeroinitializer, <vscale x 4 x ptr> align 1 [[BROADCAST_SPLAT2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: [[F_039:%.*]] = phi i8 [ 0, %[[VECTOR_PH]] ], [ [[ADD:%.*]], %[[EXIT:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = or i8 23, [[X]]
+; CHECK-NEXT: [[EXTRACT_T:%.*]] = trunc i8 [[TMP0]] to i1
+; CHECK-NEXT: br i1 [[EXTRACT_T]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: store i8 0, ptr [[DST]], align 1
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[ADD]] = add i8 [[F_039]], 1
+; CHECK-NEXT: [[CONV:%.*]] = sext i8 [[F_039]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[CONV]], 8
+; CHECK-NEXT: br i1 [[CMP]], label %[[VECTOR_BODY]], label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -214,7 +214,7 @@ define void @icmp_only_first_op_truncated(ptr noalias %dst, i32 %x, i64 %N, i64
; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP14]] to i64
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -361,15 +361,4 @@ attributes #1 = { "target-features"="+64bit,+v" }
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
-; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
-; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], !"LVerDomain"}
-; CHECK: [[META9]] = !{[[META10:![0-9]+]], [[META11:![0-9]+]]}
-; CHECK: [[META10]] = distinct !{[[META10]], [[META8]]}
-; CHECK: [[META11]] = distinct !{[[META11]], [[META8]]}
-; CHECK: [[META12]] = !{[[META10]]}
-; CHECK: [[META13]] = !{[[META11]]}
-; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
index 10bfd4e55fa0c..851e60038e85d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
@@ -14,15 +14,15 @@ define void @truncate_to_minimal_bitwidths_widen_cast_recipe(ptr %src) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 9, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> zeroinitializer, <vscale x 8 x ptr> align 1 splat (ptr null), <vscale x 8 x i1> splat (i1 true), i32 [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP7]] to i64
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv1i8.nxv1p0(<vscale x 1 x i8> zeroinitializer, <vscale x 1 x ptr> align 1 splat (ptr null), <vscale x 1 x i1> splat (i1 true), i32 [[TMP7]])
+; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
index 06d74a77bb0e3..186ad58eac1aa 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
@@ -19,8 +19,6 @@ define void @type_info_cache_clobber(ptr %dstv, ptr %src, i64 %wide.trip.count)
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[DSTV]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -28,14 +26,16 @@ define void @type_info_cache_clobber(ptr %dstv, ptr %src, i64 %wide.trip.count)
; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[EVL_BASED_IV]]
; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i8> @llvm.vp.load.nxv8i8.p0(ptr align 1 [[TMP13]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP15:%.*]] = zext <vscale x 8 x i8> [[VP_OP_LOAD]] to <vscale x 8 x i32>
-; CHECK-NEXT: [[TMP23:%.*]] = ashr <vscale x 8 x i32> [[TMP15]], zeroinitializer
-; CHECK-NEXT: [[TMP16:%.*]] = icmp ult <vscale x 8 x i32> [[TMP15]], zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = select <vscale x 8 x i1> [[TMP16]], <vscale x 8 x i32> [[TMP23]], <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP24:%.*]] = trunc <vscale x 8 x i32> [[TMP17]] to <vscale x 8 x i8>
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP24]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 splat (ptr null), <vscale x 8 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext <vscale x 8 x i8> [[VP_OP_LOAD]] to <vscale x 8 x i32>
+; CHECK-NEXT: [[TMP8:%.*]] = ashr <vscale x 8 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ult <vscale x 8 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = select <vscale x 8 x i1> [[TMP6]], <vscale x 8 x i32> [[TMP8]], <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = trunc <vscale x 8 x i32> [[TMP7]] to <vscale x 8 x i8>
; CHECK-NEXT: [[TMP20:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT: [[TMP10:%.*]] = sub i64 [[TMP20]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 8 x i8> [[TMP24]], i64 [[TMP10]]
+; CHECK-NEXT: store i8 [[TMP12]], ptr [[DSTV]], align 1, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: store i16 0, ptr null, align 2
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP20]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP20]]
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
index e068bf50506f4..c578e0c26e884 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
@@ -549,26 +549,26 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
; SCALABLE-NEXT: [[ENTRY:.*:]]
; SCALABLE-NEXT: br label %[[VECTOR_PH:.*]]
; SCALABLE: [[VECTOR_PH]]:
-; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[B]], i64 0
-; SCALABLE-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; SCALABLE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; SCALABLE-NEXT: [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; SCALABLE: [[VECTOR_BODY]]:
; SCALABLE-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; SCALABLE-NEXT: [[TMP8:%.*]] = zext i32 [[TMP7]] to i64
-; SCALABLE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
-; SCALABLE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; SCALABLE-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT1]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
+; SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; SCALABLE-NEXT: [[TMP3:%.*]] = sub i64 [[TMP8]], 1
+; SCALABLE-NEXT: [[TMP4:%.*]] = extractelement <vscale x 2 x i64> [[VEC_IND]], i64 [[TMP3]]
+; SCALABLE-NEXT: store i64 [[TMP4]], ptr [[B]], align 8
; SCALABLE-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
; SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP16]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[TMP10]]
; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
-; SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; SCALABLE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; SCALABLE-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; SCALABLE: [[MIDDLE_BLOCK]]:
@@ -620,23 +620,23 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
; TF-SCALABLE-NEXT: [[ENTRY:.*:]]
; TF-SCALABLE-NEXT: br label %[[VECTOR_PH:.*]]
; TF-SCALABLE: [[VECTOR_PH]]:
-; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[B]], i64 0
-; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
-; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
-; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
+; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; TF-SCALABLE-NEXT: [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; TF-SCALABLE: [[VECTOR_BODY]]:
; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; TF-SCALABLE-NEXT: [[TMP13:%.*]] = zext i32 [[TMP9]] to i64
; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; TF-SCALABLE-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
+; TF-SCALABLE-NEXT: [[TMP3:%.*]] = sub i64 [[TMP13]], 1
+; TF-SCALABLE-NEXT: [[TMP4:%.*]] = extractelement <vscale x 2 x i64> [[VEC_IND]], i64 [[TMP3]]
+; TF-SCALABLE-NEXT: store i64 [[TMP4]], ptr [[B]], align 8
; TF-SCALABLE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT3]], ptr align 8 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
+; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; TF-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
@@ -898,191 +898,6 @@ for.end:
ret void
}
-define void @uniform_load_store(ptr %p, ptr %q, i32 %n) {
-; SCALABLE-LABEL: define void @uniform_load_store(
-; SCALABLE-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
-; SCALABLE-NEXT: [[ENTRY:.*:]]
-; SCALABLE-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; SCALABLE: [[VECTOR_SCEVCHECK]]:
-; SCALABLE-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
-; SCALABLE-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
-; SCALABLE-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; SCALABLE: [[VECTOR_MEMCHECK]]:
-; SCALABLE-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[Q]], i64 4
-; SCALABLE-NEXT: [[TMP2:%.*]] = add i32 [[N]], -1
-; SCALABLE-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
-; SCALABLE-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
-; SCALABLE-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 4
-; SCALABLE-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
-; SCALABLE-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP1]]
-; SCALABLE-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[P]], [[SCEVGEP]]
-; SCALABLE-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; SCALABLE-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; SCALABLE: [[VECTOR_PH]]:
-; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[Q]], i64 0
-; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
-; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
-; SCALABLE: [[VECTOR_BODY]]:
-; SCALABLE-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; SCALABLE-NEXT: [[AVL:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; SCALABLE-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[EVL_BASED_IV]]
-; SCALABLE-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META10:![0-9]+]]
-; SCALABLE-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 1)
-; SCALABLE-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META10]]
-; SCALABLE-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP8]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META13:![0-9]+]], !noalias [[META10]]
-; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add i32 [[TMP6]], [[EVL_BASED_IV]]
-; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
-; SCALABLE-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; SCALABLE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; SCALABLE: [[MIDDLE_BLOCK]]:
-; SCALABLE-NEXT: br label %[[EXIT:.*]]
-; SCALABLE: [[SCALAR_PH]]:
-; SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; SCALABLE-NEXT: br label %[[LOOP:.*]]
-; SCALABLE: [[LOOP]]:
-; SCALABLE-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; SCALABLE-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
-; SCALABLE-NEXT: [[X:%.*]] = load i32, ptr [[GEP]], align 4
-; SCALABLE-NEXT: [[Y:%.*]] = add i32 [[X]], 1
-; SCALABLE-NEXT: store i32 [[Y]], ptr [[GEP]], align 4
-; SCALABLE-NEXT: store i32 [[Y]], ptr [[Q]], align 4
-; SCALABLE-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
-; SCALABLE-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; SCALABLE-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
-; SCALABLE: [[EXIT]]:
-; SCALABLE-NEXT: ret void
-;
-; FIXEDLEN-LABEL: define void @uniform_load_store(
-; FIXEDLEN-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
-; FIXEDLEN-NEXT: [[ENTRY:.*]]:
-; FIXEDLEN-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 17
-; FIXEDLEN-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; FIXEDLEN: [[VECTOR_SCEVCHECK]]:
-; FIXEDLEN-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
-; FIXEDLEN-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
-; FIXEDLEN-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
-; FIXEDLEN: [[VECTOR_MEMCHECK]]:
-; FIXEDLEN-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[Q]], i64 4
-; FIXEDLEN-NEXT: [[TMP2:%.*]] = add i32 [[N]], -1
-; FIXEDLEN-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
-; FIXEDLEN-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
-; FIXEDLEN-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 4
-; FIXEDLEN-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
-; FIXEDLEN-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP1]]
-; FIXEDLEN-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[P]], [[SCEVGEP]]
-; FIXEDLEN-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; FIXEDLEN-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; FIXEDLEN: [[VECTOR_PH]]:
-; FIXEDLEN-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
-; FIXEDLEN-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
-; FIXEDLEN-NEXT: br label %[[VECTOR_BODY:.*]]
-; FIXEDLEN: [[VECTOR_BODY]]:
-; FIXEDLEN-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; FIXEDLEN-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
-; FIXEDLEN-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i64 8
-; FIXEDLEN-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP6]], align 4, !alias.scope [[META18:![0-9]+]]
-; FIXEDLEN-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i32>, ptr [[TMP7]], align 4, !alias.scope [[META18]]
-; FIXEDLEN-NEXT: [[TMP8:%.*]] = add <8 x i32> [[WIDE_LOAD]], splat (i32 1)
-; FIXEDLEN-NEXT: [[TMP9:%.*]] = add <8 x i32> [[WIDE_LOAD2]], splat (i32 1)
-; FIXEDLEN-NEXT: store <8 x i32> [[TMP8]], ptr [[TMP6]], align 4, !alias.scope [[META18]]
-; FIXEDLEN-NEXT: store <8 x i32> [[TMP9]], ptr [[TMP7]], align 4, !alias.scope [[META18]]
-; FIXEDLEN-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP9]], i32 7
-; FIXEDLEN-NEXT: store i32 [[TMP10]], ptr [[Q]], align 4, !alias.scope [[META21:![0-9]+]], !noalias [[META18]]
-; FIXEDLEN-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; FIXEDLEN-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; FIXEDLEN-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
-; FIXEDLEN: [[MIDDLE_BLOCK]]:
-; FIXEDLEN-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; FIXEDLEN-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; FIXEDLEN: [[SCALAR_PH]]:
-; FIXEDLEN-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; FIXEDLEN-NEXT: br label %[[LOOP:.*]]
-; FIXEDLEN: [[LOOP]]:
-; FIXEDLEN-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; FIXEDLEN-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
-; FIXEDLEN-NEXT: [[X:%.*]] = load i32, ptr [[GEP]], align 4
-; FIXEDLEN-NEXT: [[Y:%.*]] = add i32 [[X]], 1
-; FIXEDLEN-NEXT: store i32 [[Y]], ptr [[GEP]], align 4
-; FIXEDLEN-NEXT: store i32 [[Y]], ptr [[Q]], align 4
-; FIXEDLEN-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
-; FIXEDLEN-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; FIXEDLEN-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
-; FIXEDLEN: [[EXIT]]:
-; FIXEDLEN-NEXT: ret void
-;
-; TF-SCALABLE-LABEL: define void @uniform_load_store(
-; TF-SCALABLE-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
-; TF-SCALABLE-NEXT: [[ENTRY:.*:]]
-; TF-SCALABLE-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; TF-SCALABLE: [[VECTOR_SCEVCHECK]]:
-; TF-SCALABLE-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
-; TF-SCALABLE-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
-; TF-SCALABLE-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; TF-SCALABLE: [[VECTOR_MEMCHECK]]:
-; TF-SCALABLE-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[Q]], i64 4
-; TF-SCALABLE-NEXT: [[TMP2:%.*]] = add i32 [[N]], -1
-; TF-SCALABLE-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
-; TF-SCALABLE-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
-; TF-SCALABLE-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 4
-; TF-SCALABLE-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
-; TF-SCALABLE-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP1]]
-; TF-SCALABLE-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[P]], [[SCEVGEP]]
-; TF-SCALABLE-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; TF-SCALABLE-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; TF-SCALABLE: [[VECTOR_PH]]:
-; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[Q]], i64 0
-; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
-; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF-SCALABLE: [[VECTOR_BODY]]:
-; TF-SCALABLE-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-SCALABLE-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[EVL_BASED_IV]]
-; TF-SCALABLE-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META10:![0-9]+]]
-; TF-SCALABLE-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 1)
-; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META10]]
-; TF-SCALABLE-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP8]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]]), !alias.scope [[META13:![0-9]+]], !noalias [[META10]]
-; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add i32 [[TMP6]], [[EVL_BASED_IV]]
-; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
-; TF-SCALABLE-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; TF-SCALABLE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; TF-SCALABLE: [[MIDDLE_BLOCK]]:
-; TF-SCALABLE-NEXT: br label %[[EXIT:.*]]
-; TF-SCALABLE: [[SCALAR_PH]]:
-; TF-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; TF-SCALABLE-NEXT: br label %[[LOOP:.*]]
-; TF-SCALABLE: [[LOOP]]:
-; TF-SCALABLE-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; TF-SCALABLE-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
-; TF-SCALABLE-NEXT: [[X:%.*]] = load i32, ptr [[GEP]], align 4
-; TF-SCALABLE-NEXT: [[Y:%.*]] = add i32 [[X]], 1
-; TF-SCALABLE-NEXT: store i32 [[Y]], ptr [[GEP]], align 4
-; TF-SCALABLE-NEXT: store i32 [[Y]], ptr [[Q]], align 4
-; TF-SCALABLE-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
-; TF-SCALABLE-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; TF-SCALABLE-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
-; TF-SCALABLE: [[EXIT]]:
-; TF-SCALABLE-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i32 [0, %entry], [%iv.next, %loop]
- %gep = getelementptr i32, ptr %p, i32 %iv
- %x = load i32, ptr %gep
- %y = add i32 %x, 1
- store i32 %y, ptr %gep
- store i32 %y, ptr %q
- %iv.next = add i32 %iv, 1
- %done = icmp eq i32 %iv.next, %n
- br i1 %done, label %exit, label %loop
-
-exit:
- ret void
-}
-
!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.width", i32 4}
!2 = !{!"llvm.loop.vectorize.enable"}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
index dca4f47738309..0ecb9f53eb3fc 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
@@ -18,15 +18,15 @@ define void @vf_will_not_generate_any_vector_insts(ptr %src, ptr %dst) {
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4, !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT3]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv1i32.nxv1p0(<vscale x 1 x i32> [[BROADCAST_SPLAT3]], <vscale x 1 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP5]] to i64
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
>From 0fea254cf6efba3e6633dfed3b26c12c640d6a49 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 29 Jun 2026 22:41:13 -0700
Subject: [PATCH 3/6] Remove cost model decision since all scalar load sink to
middle.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 9 --
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 86 ++++++-----------
.../Transforms/Vectorize/VPlanTransforms.cpp | 92 ++++++-------------
.../Transforms/Vectorize/VPlanTransforms.h | 5 -
.../Transforms/LoopVectorize/RISCV/divrem.ll | 10 +-
.../RISCV/gather-scatter-cost.ll | 10 +-
.../LoopVectorize/RISCV/induction-costs.ll | 12 +--
.../RISCV/pointer-induction-rv32.ll | 6 +-
.../LoopVectorize/RISCV/pointer-induction.ll | 13 +--
.../Transforms/LoopVectorize/RISCV/pr88802.ll | 10 +-
.../RISCV/strided-access-wide-stride.ll | 11 +--
.../truncate-to-minimal-bitwidth-evl-crash.ll | 31 +++----
.../vf-will-not-generate-any-vector-insts.ll | 41 +--------
14 files changed, 109 insertions(+), 235 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 911a4e621a234..3c65eda187a6e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6641,6 +6641,8 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
// Now optimize the initial VPlan.
RUN_VPLAN_PASS(VPlanTransforms::hoistPredicatedLoads, *Plan, PSE, OrigLoop);
RUN_VPLAN_PASS(VPlanTransforms::sinkPredicatedStores, *Plan, PSE, OrigLoop);
+ RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
+ Config.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
if (CM.foldTailWithEVL()) {
@@ -6891,12 +6893,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
if (Range.Start.isScalar())
Range.End = Range.Start * 2;
- RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
- Config.getMinimalBitwidths());
- RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *Plan);
- RUN_VPLAN_PASS(VPlanTransforms::narrowScatters, *Plan, CostCtx, Range,
- CM.foldTailWithEVL());
-
for (ElementCount VF : Range)
Plan->addVF(VF);
Plan->setName("Initial VPlan");
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1b8b6bedf6b77..814b77a96e825 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1524,15 +1524,6 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Set the symbolic name for the VPInstruction.
void setName(StringRef NewName) { Name = NewName.str(); }
- /// Compute the cost for a recipe with \p VF using \p Opcode, \p ValTy.
- /// This function may not be as accurate as
- /// `getCostForRecipeWithOpcode` since it only provides type-based queries.
- static InstructionCost getCostForRecipeWithOpcodeAndTypes(unsigned Opcode,
- Type *ValTy,
- Instruction *I,
- ElementCount VF,
- VPCostContext &Ctx);
-
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the VPInstruction to \p O.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b8118ed558f87..a5343caccc992 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1114,57 +1114,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
}
-InstructionCost VPInstruction::getCostForRecipeWithOpcodeAndTypes(
- unsigned Opcode, Type *ValTy, Instruction *I, ElementCount VF,
- VPCostContext &Ctx) {
- switch (Opcode) {
- case VPInstruction::LastActiveLane: {
- // LastActiveLane computes the index of the last active lane in a
- // predicate mask: NOT + cttz_elts + SUB.
- if (VF.isScalar())
- return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ValTy,
- CmpInst::makeCmpResultType(ValTy),
- CmpInst::ICMP_EQ, Ctx.CostKind);
- auto *PredTy = toVectorTy(ValTy, VF);
- IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts,
- Type::getInt64Ty(Ctx.LLVMCtx),
- {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
- InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
- // Add cost of NOT operation on the predicate.
- Cost += Ctx.TTI.getArithmeticInstrCost(
- Instruction::Xor, PredTy, Ctx.CostKind,
- {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
- {TargetTransformInfo::OK_UniformConstantValue,
- TargetTransformInfo::OP_None});
- // Add cost of SUB operation on the index.
- Cost += Ctx.TTI.getArithmeticInstrCost(
- Instruction::Sub, Type::getInt64Ty(Ctx.LLVMCtx), Ctx.CostKind);
- return Cost;
- }
- case VPInstruction::ExtractLane: {
- // Compute the cost of ExtractLane for a vector with the given scalar
- // element type. ExtractLane extracts an element at a runtime-determined
- // index.
- if (VF.isScalar())
- return 0;
- return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement,
- toVectorTy(ValTy, VF), Ctx.CostKind);
- }
- case Instruction::Store: {
- assert(I && "Querying cost for store must have underlying instruction.");
- return Ctx.TTI.getMemoryOpCost(Opcode, ValTy, getLoadStoreAlignment(I),
- getLoadStoreAddressSpace(I), Ctx.CostKind,
- TTI::getOperandInfo(I->getOperand(0)), I);
- }
- case Instruction::Sub: {
- return Ctx.TTI.getArithmeticInstrCost(Opcode, toVectorTy(ValTy, VF),
- Ctx.CostKind);
- }
- default:
- llvm_unreachable("Unsupported opcode");
- }
-}
-
InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
unsigned Opcode, ElementCount VF, VPCostContext &Ctx) const {
Type *ScalarTy = this->getScalarType();
@@ -1400,8 +1349,16 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
}
case Instruction::ExtractElement:
case VPInstruction::ExtractLane: {
- return getCostForRecipeWithOpcodeAndTypes(
- getOpcode(), getOperand(0)->getScalarType(), nullptr, VF, Ctx);
+ if (VF.isScalar()) {
+ // ExtractLane with VF=1 takes care of handling extracting across multiple
+ // parts.
+ return 0;
+ }
+
+ // Add on the cost of extracting the element.
+ auto *VecTy = toVectorTy(getOperand(0)->getScalarType(), VF);
+ return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
+ Ctx.CostKind);
}
case VPInstruction::AnyOf: {
auto *VecTy = toVectorTy(this->getScalarType(), VF);
@@ -1422,9 +1379,26 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
}
case VPInstruction::LastActiveLane: {
- return getCostForRecipeWithOpcodeAndTypes(VPInstruction::LastActiveLane,
- getOperand(0)->getScalarType(),
- nullptr, VF, Ctx);
+ Type *Ty = this->getScalarType();
+ Type *ScalarTy = getOperand(0)->getScalarType();
+ if (VF.isScalar())
+ return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy,
+ CmpInst::makeCmpResultType(ScalarTy),
+ CmpInst::ICMP_EQ, Ctx.CostKind);
+ // Calculate the cost of determining the lane index: NOT + cttz_elts + SUB.
+ auto *PredTy = toVectorTy(ScalarTy, VF);
+ IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts, Ty,
+ {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
+ InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ // Add cost of NOT operation on the predicate.
+ Cost += Ctx.TTI.getArithmeticInstrCost(
+ Instruction::Xor, PredTy, Ctx.CostKind,
+ {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+ {TargetTransformInfo::OK_UniformConstantValue,
+ TargetTransformInfo::OP_None});
+ // Add cost of SUB operation on the index.
+ Cost += Ctx.TTI.getArithmeticInstrCost(Instruction::Sub, Ty, Ctx.CostKind);
+ return Cost;
}
case VPInstruction::ExtractLastActive: {
Type *ScalarTy = this->getScalarType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9f8a1cc149713..8937fbb134914 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1733,71 +1733,6 @@ getUnmaskedDivRemOpcode(Intrinsic::ID ID) {
}
}
-void VPlanTransforms::narrowScatters(VPlan &Plan, VPCostContext &Ctx,
- VFRange &Range, bool FoldTailWithEVL) {
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
- for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- // Convert an unmasked or header masked scatter with a uniform address
- // into extract-last-lane + scalar store.
- auto *WidenStoreR = dyn_cast<VPWidenStoreRecipe>(&R);
- if (!WidenStoreR ||
- !vputils::isUniformAcrossVFsAndUFs(WidenStoreR->getAddr()) ||
- WidenStoreR->isConsecutive())
- continue;
-
- // Convert the scatter to a scalar store if it is header masked.
- VPValue *Mask = WidenStoreR->getMask();
- if (!Mask || !match(Mask, m_HeaderMask()))
- continue;
-
- // If the body is header-masked, it guarantees each iteration has at
- // least one active lane. So it is safe to convert the scatter to a
- // scalar store.
- if (!LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](ElementCount VF) {
- InstructionCost ScatterCost = WidenStoreR->computeCost(VF, Ctx);
- Type *ValTy = WidenStoreR->getStoredValue()->getScalarType();
-
- // ScalarCost = LastActiveLaneCost + ExtractLaneCost +
- // ScalarStoreCost.
- InstructionCost ScalarCost = 0;
-
- // LastActiveLane can lower to EVL - 1 when folding tail with
- // EVL.
- ScalarCost +=
- FoldTailWithEVL
- ? VPInstruction::getCostForRecipeWithOpcodeAndTypes(
- Instruction::Sub, Type::getInt32Ty(Ctx.LLVMCtx),
- nullptr, ElementCount::getFixed(1), Ctx)
- : VPInstruction::getCostForRecipeWithOpcodeAndTypes(
- VPInstruction::LastActiveLane,
- Type::getInt1Ty(Ctx.LLVMCtx), nullptr, VF, Ctx);
- ScalarCost += VPInstruction::getCostForRecipeWithOpcodeAndTypes(
- VPInstruction::ExtractLane, ValTy, nullptr, VF, Ctx);
- ScalarCost += VPInstruction::getCostForRecipeWithOpcodeAndTypes(
- Instruction::Store, ValTy, &WidenStoreR->getIngredient(),
- VF, Ctx);
-
- return ScalarCost <= ScatterCost;
- },
- Range))
- continue;
- VPBuilder Builder(WidenStoreR);
- VPInstruction *LastActiveLane = Builder.createLastActiveLane(Mask);
- VPInstruction *Extract =
- Builder.createNaryOp(VPInstruction::ExtractLane,
- {LastActiveLane, WidenStoreR->getStoredValue()});
- auto *ScalarStore = new VPReplicateRecipe(
- &WidenStoreR->getIngredient(), {Extract, WidenStoreR->getAddr()},
- /*IsSingleScalar*/ true, /*Mask*/ nullptr, {},
- /*Metadata*/ *WidenStoreR);
- ScalarStore->insertBefore(WidenStoreR);
- WidenStoreR->eraseFromParent();
- }
- }
-}
-
static void narrowToSingleScalarRecipes(VPlan &Plan) {
if (Plan.hasScalarVFOnly())
return;
@@ -1806,8 +1741,33 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
vp_depth_first_deep(Plan.getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
if (!isa<VPWidenRecipe, VPWidenGEPRecipe, VPReplicateRecipe,
- VPWidenIntrinsicRecipe>(&R))
+ VPWidenIntrinsicRecipe, VPWidenStoreRecipe>(&R))
+ continue;
+
+ // Narrow header masked scatter to scalar store.
+ auto *WidenStoreR = dyn_cast<VPWidenStoreRecipe>(&R);
+ if (WidenStoreR) {
+ if (!vputils::isUniformAcrossVFsAndUFs(WidenStoreR->getAddr()) ||
+ WidenStoreR->isConsecutive())
+ continue;
+ VPValue *Mask = WidenStoreR->getMask();
+ if (!Mask || !match(Mask, m_HeaderMask()))
+ continue;
+
+ VPBuilder Builder(WidenStoreR);
+ VPInstruction *LastActiveLane = Builder.createLastActiveLane(Mask);
+ VPInstruction *Extract = Builder.createNaryOp(
+ VPInstruction::ExtractLane,
+ {LastActiveLane, WidenStoreR->getStoredValue()});
+ auto *ScalarStore = new VPReplicateRecipe(
+ &WidenStoreR->getIngredient(), {Extract, WidenStoreR->getAddr()},
+ /*IsSingleScalar*/ true, /*Mask*/ nullptr, {},
+ /*Metadata*/ *WidenStoreR);
+ ScalarStore->insertBefore(WidenStoreR);
+ WidenStoreR->eraseFromParent();
continue;
+ }
+
auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 6a74fa9577d08..af8f6de26e0f9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -599,11 +599,6 @@ struct VPlanTransforms {
static void makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
VPCostContext &CostCtx);
-
- /// Convert the scatter to extract-last-active-lane + scalar store if
- /// profitable.
- static void narrowScatters(VPlan &Plan, VPCostContext &Ctx, VFRange &Range,
- bool FoldTailWithEVL);
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index 6536d5372bf41..f85278d203bbf 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -650,6 +650,11 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT6]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: middle.block:
; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i8> @llvm.vp.udiv.nxv4i8(<vscale x 4 x i8> [[VEC_IND]], <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 4 x i8> [[TMP4]] to <vscale x 4 x i16>
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i16> @llvm.vp.sdiv.nxv4i16(<vscale x 4 x i16> [[TMP5]], <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
@@ -659,11 +664,6 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 1
; CHECK-NEXT: [[MERGE:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI]], i64 [[TMP9]]
; CHECK-NEXT: store i32 [[MERGE]], ptr [[P:%.*]], align 4
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT6]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK: middle.block:
; CHECK-NEXT: br label [[EXIT1:%.*]]
; CHECK: exit:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
index 75b388187a194..ccbca35b6f624 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
@@ -38,15 +38,15 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP10]], i32 [[TMP11]]), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP11]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: middle.block:
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[COND]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: [[TMP13:%.*]] = sub i64 [[TMP12]], 1
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI]], i64 [[TMP13]]
-; CHECK-NEXT: store i32 [[TMP14]], ptr [[DST]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP11]]
-; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: store i32 [[TMP14]], ptr [[DST]], align 4, !alias.scope [[META6:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: scalar.ph:
; CHECK-NEXT: br label [[LOOP:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
index f7686fadedc32..bbe4180368a55 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
@@ -139,18 +139,18 @@ define void @test_3_inductions(ptr noalias %dst, ptr noalias %src, i64 %n) #1 {
; CHECK-NEXT: [[TMP4:%.*]] = shl i32 [[TMP3]], 1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP4]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = or <vscale x 2 x i32> [[VEC_IND1]], [[VEC_IND]]
-; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 2 x i32> [[TMP5]] to <vscale x 2 x i64>
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], <vscale x 2 x i64> [[TMP6]]
-; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP8]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x ptr> [[TMP7]], i64 [[TMP11]]
-; CHECK-NEXT: store ptr [[TMP10]], ptr [[DST]], align 8
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[VEC_IND_NEXT2]] = add <vscale x 2 x i32> [[VEC_IND1]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = or <vscale x 2 x i32> [[VEC_IND1]], [[VEC_IND]]
+; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 2 x i32> [[TMP5]] to <vscale x 2 x i64>
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], <vscale x 2 x i64> [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP8]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x ptr> [[TMP7]], i64 [[TMP11]]
+; CHECK-NEXT: store ptr [[TMP10]], ptr [[DST]], align 8
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction-rv32.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction-rv32.ll
index e5153ab3673b0..7dd5d1a5df1c9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction-rv32.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction-rv32.ll
@@ -16,14 +16,14 @@ define i32 @widenpointerinduction_evl(ptr noalias %p) {
; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i32> [[TMP0]]
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[TMP1]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x ptr> [[VECTOR_GEP]], i32 [[TMP4]]
-; CHECK-NEXT: store ptr [[TMP3]], ptr [[P]], align 4
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[TMP1]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <vscale x 4 x ptr> [[VECTOR_GEP]], i32 [[TMP3]]
+; CHECK-NEXT: store ptr [[TMP4]], ptr [[P]], align 4
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index bf29404fc6b68..fb057e1a97fd7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -28,8 +28,8 @@ define void @ptr_induction(ptr %p, ptr noalias %q, ptr noalias %p.end) #0 {
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], 1
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i64> [[TMP9]], i64 [[TMP6]]
; CHECK-NEXT: store i64 [[TMP10]], ptr [[Q]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i64> [[VEC_IND]], i64 [[TMP6]]
-; CHECK-NEXT: store i64 [[TMP8]], ptr [[P]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <vscale x 2 x i64> [[VEC_IND]], i64 [[TMP6]]
+; CHECK-NEXT: store i64 [[TMP11]], ptr [[P]], align 8
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[PTR_IND7]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP5]]
@@ -71,7 +71,7 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[START]], i64 4
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[END]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -89,7 +89,9 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[TMP18]] to <vscale x 2 x i64>
; CHECK-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
; CHECK-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP21]], <vscale x 2 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT: [[TMP29:%.*]] = sub i64 [[TMP26]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <vscale x 2 x i64> [[TMP21]], i64 [[TMP29]]
+; CHECK-NEXT: store i64 [[TMP16]], ptr [[DST]], align 1
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP26]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
; CHECK-NEXT: [[TMP27:%.*]] = mul i64 12, [[TMP26]]
@@ -98,8 +100,7 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP30:%.*]] = getelementptr nusw i8, <vscale x 2 x ptr> [[VECTOR_GEP]], i64 12
-; CHECK-NEXT: [[TMP17:%.*]] = icmp eq <vscale x 2 x ptr> [[TMP30]], [[BROADCAST_SPLAT7]]
-; CHECK-NEXT: [[TMP29:%.*]] = sub i64 [[TMP26]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = icmp eq <vscale x 2 x ptr> [[TMP30]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP25:%.*]] = extractelement <vscale x 2 x i1> [[TMP17]], i64 [[TMP29]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
index 10ca2c2ae83da..e8f4c0fdf13ef 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
@@ -23,6 +23,11 @@ define void @test(ptr %p, i64 %a, i8 %b) {
; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[TMP11]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP11]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_COND]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
; CHECK-NEXT: [[TMP6:%.*]] = icmp slt <vscale x 8 x i32> [[VEC_IND]], splat (i32 2)
; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP6]], <vscale x 8 x i32> [[TMP3]], <vscale x 8 x i32> [[TMP2]]
; CHECK-NEXT: [[TMP7:%.*]] = shl <vscale x 8 x i32> [[PREDPHI]], splat (i32 8)
@@ -31,11 +36,6 @@ define void @test(ptr %p, i64 %a, i8 %b) {
; CHECK-NEXT: [[TMP10:%.*]] = sub i64 [[TMP9]], 1
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 8 x i8> [[TMP8]], i64 [[TMP10]]
; CHECK-NEXT: store i8 [[TMP12]], ptr [[P]], align 1
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP11]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
-; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_COND]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: exit:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
index 328fd13f93eed..3dee90277b6a0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
@@ -9,9 +9,6 @@ define void @stride_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst,
; CHECK-LABEL: define void @stride_exceeds_i32_max(
; CHECK-SAME: ptr noalias readonly [[SRC:%.*]], ptr noalias [[DST:%.*]], i16 [[START:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = sext i16 [[START]] to i64
-; CHECK-NEXT: [[TMP1:%.*]] = mul nsw i64 [[TMP0]], 3000000000
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i16 -1, [[START]]
; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 1
@@ -23,8 +20,6 @@ define void @stride_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst,
; CHECK-NEXT: [[TMP12:%.*]] = sext i16 [[START]] to i64
; CHECK-NEXT: [[TMP13:%.*]] = mul nsw i64 [[TMP12]], 3000000000
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 16 x ptr> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -35,9 +30,9 @@ define void @stride_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst,
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP11]]
; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP8]], i64 3000000000, <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
; CHECK-NEXT: [[TMP15:%.*]] = zext i32 [[TMP6]] to i64
-; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[TMP15]], 1
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 8 x i8> [[TMP14]], i64 [[TMP12]]
-; CHECK-NEXT: store i8 [[TMP13]], ptr [[DST]], align 1
+; CHECK-NEXT: [[TMP16:%.*]] = sub i64 [[TMP15]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <vscale x 8 x i8> [[TMP14]], i64 [[TMP16]]
+; CHECK-NEXT: store i8 [[TMP17]], ptr [[DST]], align 1
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP6]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
index 851e60038e85d..1338530b1f63c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
@@ -8,20 +8,20 @@
define void @truncate_to_minimal_bitwidths_widen_cast_recipe(ptr %src) {
; CHECK-LABEL: define void @truncate_to_minimal_bitwidths_widen_cast_recipe(
; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[VECTOR_PH:.*]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 9, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv1i8.nxv1p0(<vscale x 1 x i8> zeroinitializer, <vscale x 1 x ptr> align 1 splat (ptr null), <vscale x 1 x i1> splat (i1 true), i32 [[TMP7]])
-; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP7]] to i64
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP1]]
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[GEP_SRC]], align 1
+; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT: [[MUL16:%.*]] = mul i32 0, [[CONV]]
+; CHECK-NEXT: [[SHR35:%.*]] = lshr i32 [[MUL16]], 1
+; CHECK-NEXT: [[CONV36:%.*]] = trunc i32 [[SHR35]] to i8
+; CHECK-NEXT: store i8 [[CONV36]], ptr null, align 1
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 8
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
@@ -71,7 +71,7 @@ define void @truncate_i16_to_i8_cse(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: store i8 [[TMP10]], ptr [[DST]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4294967296
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -88,7 +88,7 @@ define void @truncate_i16_to_i8_cse(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: [[COUNT_NEXT]] = add i32 [[COUNT]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[COUNT_NEXT]], 0
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -117,6 +117,5 @@ exit:
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
index 0ecb9f53eb3fc..4d6c803355b16 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
@@ -7,33 +7,7 @@ target triple = "riscv64-unknown-unknown-elf"
define void @vf_will_not_generate_any_vector_insts(ptr %src, ptr %dst) {
; CHECK-LABEL: define void @vf_will_not_generate_any_vector_insts(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
-; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 4
-; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4, !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv1i32.nxv1p0(<vscale x 1 x i32> [[BROADCAST_SPLAT3]], <vscale x 1 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP5]] to i64
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
@@ -41,7 +15,7 @@ define void @vf_will_not_generate_any_vector_insts(ptr %src, ptr %dst) {
; CHECK-NEXT: store i32 [[DOTPRE]], ptr [[DST]], align 4
; CHECK-NEXT: [[TMP3]] = add nuw i64 [[TMP2]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[TMP3]], 100
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -59,14 +33,3 @@ loop:
exit:
ret void
}
-;.
-; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
-; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
-; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
-; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
-; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
-; CHECK: [[META6]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META6]]}
-;.
>From e54d144ff9715f5bab6457df32482807fe4bcb1d Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 3 Aug 2026 22:11:34 -0700
Subject: [PATCH 4/6] Revert "Remove cost model decision since all scalar load
sink to middle."
This reverts commit 7bf6703e4bb83f9972a62099abe3f309e93e7c17.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 9 ++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 86 +++++++++++------
.../Transforms/Vectorize/VPlanTransforms.cpp | 92 +++++++++++++------
.../Transforms/Vectorize/VPlanTransforms.h | 5 +
.../LoopVectorize/RISCV/pointer-induction.ll | 4 +-
.../RISCV/strided-access-wide-stride.ll | 2 +-
.../truncate-to-minimal-bitwidth-evl-crash.ll | 31 ++++---
.../vf-will-not-generate-any-vector-insts.ll | 41 ++++++++-
.../VPlan/vplan-print-before-after-all.ll | 4 +-
10 files changed, 203 insertions(+), 79 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3c65eda187a6e..911a4e621a234 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6641,8 +6641,6 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
// Now optimize the initial VPlan.
RUN_VPLAN_PASS(VPlanTransforms::hoistPredicatedLoads, *Plan, PSE, OrigLoop);
RUN_VPLAN_PASS(VPlanTransforms::sinkPredicatedStores, *Plan, PSE, OrigLoop);
- RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
- Config.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
if (CM.foldTailWithEVL()) {
@@ -6893,6 +6891,12 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
if (Range.Start.isScalar())
Range.End = Range.Start * 2;
+ RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
+ Config.getMinimalBitwidths());
+ RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *Plan);
+ RUN_VPLAN_PASS(VPlanTransforms::narrowScatters, *Plan, CostCtx, Range,
+ CM.foldTailWithEVL());
+
for (ElementCount VF : Range)
Plan->addVF(VF);
Plan->setName("Initial VPlan");
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 814b77a96e825..1b8b6bedf6b77 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1524,6 +1524,15 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Set the symbolic name for the VPInstruction.
void setName(StringRef NewName) { Name = NewName.str(); }
+ /// Compute the cost for a recipe with \p VF using \p Opcode, \p ValTy.
+ /// This function may not be as accurate as
+ /// `getCostForRecipeWithOpcode` since it only provides type-based queries.
+ static InstructionCost getCostForRecipeWithOpcodeAndTypes(unsigned Opcode,
+ Type *ValTy,
+ Instruction *I,
+ ElementCount VF,
+ VPCostContext &Ctx);
+
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the VPInstruction to \p O.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a5343caccc992..b8118ed558f87 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1114,6 +1114,57 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
}
+InstructionCost VPInstruction::getCostForRecipeWithOpcodeAndTypes(
+ unsigned Opcode, Type *ValTy, Instruction *I, ElementCount VF,
+ VPCostContext &Ctx) {
+ switch (Opcode) {
+ case VPInstruction::LastActiveLane: {
+ // LastActiveLane computes the index of the last active lane in a
+ // predicate mask: NOT + cttz_elts + SUB.
+ if (VF.isScalar())
+ return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ValTy,
+ CmpInst::makeCmpResultType(ValTy),
+ CmpInst::ICMP_EQ, Ctx.CostKind);
+ auto *PredTy = toVectorTy(ValTy, VF);
+ IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts,
+ Type::getInt64Ty(Ctx.LLVMCtx),
+ {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
+ InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ // Add cost of NOT operation on the predicate.
+ Cost += Ctx.TTI.getArithmeticInstrCost(
+ Instruction::Xor, PredTy, Ctx.CostKind,
+ {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+ {TargetTransformInfo::OK_UniformConstantValue,
+ TargetTransformInfo::OP_None});
+ // Add cost of SUB operation on the index.
+ Cost += Ctx.TTI.getArithmeticInstrCost(
+ Instruction::Sub, Type::getInt64Ty(Ctx.LLVMCtx), Ctx.CostKind);
+ return Cost;
+ }
+ case VPInstruction::ExtractLane: {
+ // Compute the cost of ExtractLane for a vector with the given scalar
+ // element type. ExtractLane extracts an element at a runtime-determined
+ // index.
+ if (VF.isScalar())
+ return 0;
+ return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement,
+ toVectorTy(ValTy, VF), Ctx.CostKind);
+ }
+ case Instruction::Store: {
+ assert(I && "Querying cost for store must have underlying instruction.");
+ return Ctx.TTI.getMemoryOpCost(Opcode, ValTy, getLoadStoreAlignment(I),
+ getLoadStoreAddressSpace(I), Ctx.CostKind,
+ TTI::getOperandInfo(I->getOperand(0)), I);
+ }
+ case Instruction::Sub: {
+ return Ctx.TTI.getArithmeticInstrCost(Opcode, toVectorTy(ValTy, VF),
+ Ctx.CostKind);
+ }
+ default:
+ llvm_unreachable("Unsupported opcode");
+ }
+}
+
InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
unsigned Opcode, ElementCount VF, VPCostContext &Ctx) const {
Type *ScalarTy = this->getScalarType();
@@ -1349,16 +1400,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
}
case Instruction::ExtractElement:
case VPInstruction::ExtractLane: {
- if (VF.isScalar()) {
- // ExtractLane with VF=1 takes care of handling extracting across multiple
- // parts.
- return 0;
- }
-
- // Add on the cost of extracting the element.
- auto *VecTy = toVectorTy(getOperand(0)->getScalarType(), VF);
- return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
- Ctx.CostKind);
+ return getCostForRecipeWithOpcodeAndTypes(
+ getOpcode(), getOperand(0)->getScalarType(), nullptr, VF, Ctx);
}
case VPInstruction::AnyOf: {
auto *VecTy = toVectorTy(this->getScalarType(), VF);
@@ -1379,26 +1422,9 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
}
case VPInstruction::LastActiveLane: {
- Type *Ty = this->getScalarType();
- Type *ScalarTy = getOperand(0)->getScalarType();
- if (VF.isScalar())
- return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy,
- CmpInst::makeCmpResultType(ScalarTy),
- CmpInst::ICMP_EQ, Ctx.CostKind);
- // Calculate the cost of determining the lane index: NOT + cttz_elts + SUB.
- auto *PredTy = toVectorTy(ScalarTy, VF);
- IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts, Ty,
- {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
- InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
- // Add cost of NOT operation on the predicate.
- Cost += Ctx.TTI.getArithmeticInstrCost(
- Instruction::Xor, PredTy, Ctx.CostKind,
- {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
- {TargetTransformInfo::OK_UniformConstantValue,
- TargetTransformInfo::OP_None});
- // Add cost of SUB operation on the index.
- Cost += Ctx.TTI.getArithmeticInstrCost(Instruction::Sub, Ty, Ctx.CostKind);
- return Cost;
+ return getCostForRecipeWithOpcodeAndTypes(VPInstruction::LastActiveLane,
+ getOperand(0)->getScalarType(),
+ nullptr, VF, Ctx);
}
case VPInstruction::ExtractLastActive: {
Type *ScalarTy = this->getScalarType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 8937fbb134914..9f8a1cc149713 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1733,6 +1733,71 @@ getUnmaskedDivRemOpcode(Intrinsic::ID ID) {
}
}
+void VPlanTransforms::narrowScatters(VPlan &Plan, VPCostContext &Ctx,
+ VFRange &Range, bool FoldTailWithEVL) {
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ // Convert an unmasked or header masked scatter with a uniform address
+ // into extract-last-lane + scalar store.
+ auto *WidenStoreR = dyn_cast<VPWidenStoreRecipe>(&R);
+ if (!WidenStoreR ||
+ !vputils::isUniformAcrossVFsAndUFs(WidenStoreR->getAddr()) ||
+ WidenStoreR->isConsecutive())
+ continue;
+
+ // Convert the scatter to a scalar store if it is header masked.
+ VPValue *Mask = WidenStoreR->getMask();
+ if (!Mask || !match(Mask, m_HeaderMask()))
+ continue;
+
+ // If the body is header-masked, it guarantees each iteration has at
+ // least one active lane. So it is safe to convert the scatter to a
+ // scalar store.
+ if (!LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) {
+ InstructionCost ScatterCost = WidenStoreR->computeCost(VF, Ctx);
+ Type *ValTy = WidenStoreR->getStoredValue()->getScalarType();
+
+ // ScalarCost = LastActiveLaneCost + ExtractLaneCost +
+ // ScalarStoreCost.
+ InstructionCost ScalarCost = 0;
+
+ // LastActiveLane can lower to EVL - 1 when folding tail with
+ // EVL.
+ ScalarCost +=
+ FoldTailWithEVL
+ ? VPInstruction::getCostForRecipeWithOpcodeAndTypes(
+ Instruction::Sub, Type::getInt32Ty(Ctx.LLVMCtx),
+ nullptr, ElementCount::getFixed(1), Ctx)
+ : VPInstruction::getCostForRecipeWithOpcodeAndTypes(
+ VPInstruction::LastActiveLane,
+ Type::getInt1Ty(Ctx.LLVMCtx), nullptr, VF, Ctx);
+ ScalarCost += VPInstruction::getCostForRecipeWithOpcodeAndTypes(
+ VPInstruction::ExtractLane, ValTy, nullptr, VF, Ctx);
+ ScalarCost += VPInstruction::getCostForRecipeWithOpcodeAndTypes(
+ Instruction::Store, ValTy, &WidenStoreR->getIngredient(),
+ VF, Ctx);
+
+ return ScalarCost <= ScatterCost;
+ },
+ Range))
+ continue;
+ VPBuilder Builder(WidenStoreR);
+ VPInstruction *LastActiveLane = Builder.createLastActiveLane(Mask);
+ VPInstruction *Extract =
+ Builder.createNaryOp(VPInstruction::ExtractLane,
+ {LastActiveLane, WidenStoreR->getStoredValue()});
+ auto *ScalarStore = new VPReplicateRecipe(
+ &WidenStoreR->getIngredient(), {Extract, WidenStoreR->getAddr()},
+ /*IsSingleScalar*/ true, /*Mask*/ nullptr, {},
+ /*Metadata*/ *WidenStoreR);
+ ScalarStore->insertBefore(WidenStoreR);
+ WidenStoreR->eraseFromParent();
+ }
+ }
+}
+
static void narrowToSingleScalarRecipes(VPlan &Plan) {
if (Plan.hasScalarVFOnly())
return;
@@ -1741,33 +1806,8 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
vp_depth_first_deep(Plan.getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
if (!isa<VPWidenRecipe, VPWidenGEPRecipe, VPReplicateRecipe,
- VPWidenIntrinsicRecipe, VPWidenStoreRecipe>(&R))
- continue;
-
- // Narrow header masked scatter to scalar store.
- auto *WidenStoreR = dyn_cast<VPWidenStoreRecipe>(&R);
- if (WidenStoreR) {
- if (!vputils::isUniformAcrossVFsAndUFs(WidenStoreR->getAddr()) ||
- WidenStoreR->isConsecutive())
- continue;
- VPValue *Mask = WidenStoreR->getMask();
- if (!Mask || !match(Mask, m_HeaderMask()))
- continue;
-
- VPBuilder Builder(WidenStoreR);
- VPInstruction *LastActiveLane = Builder.createLastActiveLane(Mask);
- VPInstruction *Extract = Builder.createNaryOp(
- VPInstruction::ExtractLane,
- {LastActiveLane, WidenStoreR->getStoredValue()});
- auto *ScalarStore = new VPReplicateRecipe(
- &WidenStoreR->getIngredient(), {Extract, WidenStoreR->getAddr()},
- /*IsSingleScalar*/ true, /*Mask*/ nullptr, {},
- /*Metadata*/ *WidenStoreR);
- ScalarStore->insertBefore(WidenStoreR);
- WidenStoreR->eraseFromParent();
+ VPWidenIntrinsicRecipe>(&R))
continue;
- }
-
auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index af8f6de26e0f9..6a74fa9577d08 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -599,6 +599,11 @@ struct VPlanTransforms {
static void makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
VPCostContext &CostCtx);
+
+ /// Convert the scatter to extract-last-active-lane + scalar store if
+ /// profitable.
+ static void narrowScatters(VPlan &Plan, VPCostContext &Ctx, VFRange &Range,
+ bool FoldTailWithEVL);
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index fb057e1a97fd7..8e40ea86d5dc0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -90,8 +90,8 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
; CHECK-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
; CHECK-NEXT: [[TMP29:%.*]] = sub i64 [[TMP26]], 1
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <vscale x 2 x i64> [[TMP21]], i64 [[TMP29]]
-; CHECK-NEXT: store i64 [[TMP16]], ptr [[DST]], align 1
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <vscale x 2 x i64> [[TMP21]], i64 [[TMP29]]
+; CHECK-NEXT: store i64 [[TMP22]], ptr [[DST]], align 1
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP26]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
; CHECK-NEXT: [[TMP27:%.*]] = mul i64 12, [[TMP26]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
index 3dee90277b6a0..8dd170a99f143 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
@@ -25,7 +25,7 @@ define void @stride_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst,
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP4]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[INDEX]] to i64
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP7]], 3000000000
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP11]]
; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP8]], i64 3000000000, <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
index 1338530b1f63c..851e60038e85d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
@@ -8,20 +8,20 @@
define void @truncate_to_minimal_bitwidths_widen_cast_recipe(ptr %src) {
; CHECK-LABEL: define void @truncate_to_minimal_bitwidths_widen_cast_recipe(
; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[VECTOR_PH:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[GEP_SRC]], align 1
-; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP0]] to i32
-; CHECK-NEXT: [[MUL16:%.*]] = mul i32 0, [[CONV]]
-; CHECK-NEXT: [[SHR35:%.*]] = lshr i32 [[MUL16]], 1
-; CHECK-NEXT: [[CONV36:%.*]] = trunc i32 [[SHR35]] to i8
-; CHECK-NEXT: store i8 [[CONV36]], ptr null, align 1
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 8
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 9, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv1i8.nxv1p0(<vscale x 1 x i8> zeroinitializer, <vscale x 1 x ptr> align 1 splat (ptr null), <vscale x 1 x i1> splat (i1 true), i32 [[TMP7]])
+; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
@@ -71,7 +71,7 @@ define void @truncate_i16_to_i8_cse(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: store i8 [[TMP10]], ptr [[DST]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4294967296
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -88,7 +88,7 @@ define void @truncate_i16_to_i8_cse(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: [[COUNT_NEXT]] = add i32 [[COUNT]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[COUNT_NEXT]], 0
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -117,5 +117,6 @@ exit:
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
index 4d6c803355b16..0ecb9f53eb3fc 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
@@ -7,7 +7,33 @@ target triple = "riscv64-unknown-unknown-elf"
define void @vf_will_not_generate_any_vector_insts(ptr %src, ptr %dst) {
; CHECK-LABEL: define void @vf_will_not_generate_any_vector_insts(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 4
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv1i32.nxv1p0(<vscale x 1 x i32> [[BROADCAST_SPLAT3]], <vscale x 1 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
@@ -15,7 +41,7 @@ define void @vf_will_not_generate_any_vector_insts(ptr %src, ptr %dst) {
; CHECK-NEXT: store i32 [[DOTPRE]], ptr [[DST]], align 4
; CHECK-NEXT: [[TMP3]] = add nuw i64 [[TMP2]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[TMP3]], 100
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -33,3 +59,14 @@ loop:
exit:
ret void
}
+;.
+; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
+; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
+; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
+; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
+; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
+; CHECK: [[META6]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META6]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index a7271524f0191..28edbe21aa414 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -37,10 +37,12 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::convertToAbstractRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createInterleaveGroups
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::convertToStridedAccesses
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::truncateToMinimalBitwidths
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::simplifyRecipes
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::narrowScatters
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::dropPoisonGeneratingRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::hoistPredicatedLoads
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::sinkPredicatedStores
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::truncateToMinimalBitwidths
; CHECK-BEFORE: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimize
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeRedundantInductionCasts
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] reassociateHeaderMask
>From dcc0d78223fd89255b9a2f0174a457e7d74bd8e1 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Sun, 9 Aug 2026 19:38:39 -0700
Subject: [PATCH 5/6] Fixup tests after rebase.
---
.../LoopVectorize/RISCV/induction-costs.ll | 22 +++++------
.../LoopVectorize/RISCV/pointer-induction.ll | 30 ++++++++-------
.../LoopVectorize/RISCV/strided-accesses.ll | 7 ++--
.../truncate-to-minimal-bitwidth-cost.ll | 37 ++++++++++++-------
.../truncate-to-minimal-bitwidth-evl-crash.ll | 16 ++++----
.../LoopVectorize/RISCV/uniform-load-store.ll | 32 ++++++++--------
.../vf-will-not-generate-any-vector-insts.ll | 12 +++---
7 files changed, 84 insertions(+), 72 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
index bbe4180368a55..1752ea0814fcc 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
@@ -126,31 +126,31 @@ define void @test_3_inductions(ptr noalias %dst, ptr noalias %src, i64 %n) #1 {
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
; CHECK-NEXT: [[TMP2:%.*]] = mul <vscale x 2 x i32> [[TMP1]], splat (i32 2)
; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 2 x i32> splat (i32 1), [[TMP2]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <vscale x 2 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <vscale x 2 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[TMP4:%.*]] = shl i32 [[TMP3]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP4]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = or <vscale x 2 x i32> [[VEC_IND1]], [[VEC_IND]]
+; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 2 x i32> [[TMP5]] to <vscale x 2 x i64>
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], <vscale x 2 x i64> [[TMP6]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2p0.nxv2p0(<vscale x 2 x ptr> [[TMP7]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[VEC_IND_NEXT2]] = add <vscale x 2 x i32> [[VEC_IND1]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[VEC_IND_NEXT4]] = add <vscale x 2 x i32> [[VEC_IND1]], [[BROADCAST_SPLAT3]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = or <vscale x 2 x i32> [[VEC_IND1]], [[VEC_IND]]
-; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 2 x i32> [[TMP5]] to <vscale x 2 x i64>
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], <vscale x 2 x i64> [[TMP6]]
-; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP8]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x ptr> [[TMP7]], i64 [[TMP11]]
-; CHECK-NEXT: store ptr [[TMP10]], ptr [[DST]], align 8
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index 8e40ea86d5dc0..a55985caa28ab 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -13,6 +13,10 @@ define void @ptr_induction(ptr %p, ptr noalias %q, ptr noalias %p.end) #0 {
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP8]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[Q]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[P]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -22,16 +26,13 @@ define void @ptr_induction(ptr %p, ptr noalias %q, ptr noalias %p.end) #0 {
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = ptrtoint <vscale x 2 x ptr> [[VECTOR_GEP]] to <vscale x 2 x i64>
-; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i64> [[TMP9]], i64 [[TMP6]]
-; CHECK-NEXT: store i64 [[TMP10]], ptr [[Q]], align 8
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <vscale x 2 x i64> [[VEC_IND]], i64 [[TMP6]]
-; CHECK-NEXT: store i64 [[TMP11]], ptr [[P]], align 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint <vscale x 2 x ptr> [[VECTOR_GEP]] to <vscale x 2 x i64>
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP6]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP4]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP4]])
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT6]]
; CHECK-NEXT: [[PTR_IND7]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -71,8 +72,10 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[START]], i64 4
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[END]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[DST]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[END]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT6]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_MEMCHECK]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -89,9 +92,7 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[TMP18]] to <vscale x 2 x i64>
; CHECK-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
; CHECK-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
-; CHECK-NEXT: [[TMP29:%.*]] = sub i64 [[TMP26]], 1
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <vscale x 2 x i64> [[TMP21]], i64 [[TMP29]]
-; CHECK-NEXT: store i64 [[TMP22]], ptr [[DST]], align 1
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP21]], <vscale x 2 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP26]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
; CHECK-NEXT: [[TMP27:%.*]] = mul i64 12, [[TMP26]]
@@ -100,7 +101,8 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP30:%.*]] = getelementptr nusw i8, <vscale x 2 x ptr> [[VECTOR_GEP]], i64 12
-; CHECK-NEXT: [[TMP17:%.*]] = icmp eq <vscale x 2 x ptr> [[TMP30]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP17:%.*]] = icmp eq <vscale x 2 x ptr> [[TMP30]], [[BROADCAST_SPLAT7]]
+; CHECK-NEXT: [[TMP29:%.*]] = sub i64 [[TMP26]], 1
; CHECK-NEXT: [[TMP25:%.*]] = extractelement <vscale x 2 x i1> [[TMP17]], i64 [[TMP29]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index a00f45427bbd2..5dcacebd897fb 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -1551,6 +1551,8 @@ define void @type_cache_crash(ptr noalias %a, ptr noalias %b, ptr noalias %c, i3
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i64 48
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x double> poison, double [[C2]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[C]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1567,10 +1569,7 @@ define void @type_cache_crash(ptr noalias %a, ptr noalias %b, ptr noalias %c, i3
; CHECK-NEXT: [[TMP12:%.*]] = fsub <vscale x 2 x double> [[BROADCAST_SPLAT2]], [[TMP11]]
; CHECK-NEXT: [[TMP13:%.*]] = fmul <vscale x 2 x double> [[BROADCAST_SPLAT6]], [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 2 x double> @llvm.fmuladd.nxv2f64(<vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[BROADCAST_SPLAT8]], <vscale x 2 x double> [[TMP13]])
-; CHECK-NEXT: [[TMP18:%.*]] = zext i32 [[TMP7]] to i64
-; CHECK-NEXT: [[TMP19:%.*]] = sub i64 [[TMP18]], 1
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <vscale x 2 x double> [[TMP14]], i64 [[TMP19]]
-; CHECK-NEXT: store double [[TMP17]], ptr [[C]], align 8
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP14]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP7]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index 675cc32a8459d..bd18b22134555 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -108,22 +108,22 @@ exit:
define void @truncate_to_i1_used_by_branch(i8 %x, ptr %dst) #0 {
; CHECK-LABEL: define void @truncate_to_i1_used_by_branch(
; CHECK-SAME: i8 [[X:%.*]], ptr [[DST:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: [[VECTOR_PH:.*]]:
+; CHECK-NEXT: [[VECTOR_PH:.*:]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[F_039:%.*]] = phi i8 [ 0, %[[VECTOR_PH]] ], [ [[ADD:%.*]], %[[EXIT:.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = or i8 23, [[X]]
-; CHECK-NEXT: [[EXTRACT_T:%.*]] = trunc i8 [[TMP0]] to i1
-; CHECK-NEXT: br i1 [[EXTRACT_T]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store i8 0, ptr [[DST]], align 1
-; CHECK-NEXT: br label %[[EXIT]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[ADD]] = add i8 [[F_039]], 1
-; CHECK-NEXT: [[CONV:%.*]] = sext i8 [[F_039]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[CONV]], 8
-; CHECK-NEXT: br i1 [[CMP]], label %[[VECTOR_BODY]], label %[[EXIT1:.*]]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 9, %[[VECTOR_BODY]] ], [ [[AVL_NEXT:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 1, i1 true)
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv1i8.nxv1p0(<vscale x 1 x i8> zeroinitializer, <vscale x 1 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[EXIT1:.*]], label %[[EXIT]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[EXIT1]]:
+; CHECK-NEXT: br label %[[EXIT2:.*]]
+; CHECK: [[EXIT2]]:
; CHECK-NEXT: ret void
;
entry:
@@ -214,7 +214,7 @@ define void @icmp_only_first_op_truncated(ptr noalias %dst, i32 %x, i64 %N, i64
; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP14]] to i64
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -361,4 +361,15 @@ attributes #1 = { "target-features"="+64bit,+v" }
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
+; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], !"LVerDomain"}
+; CHECK: [[META9]] = !{[[META10:![0-9]+]], [[META11:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], [[META8]]}
+; CHECK: [[META11]] = distinct !{[[META11]], [[META8]]}
+; CHECK: [[META12]] = !{[[META10]]}
+; CHECK: [[META13]] = !{[[META11]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
index 851e60038e85d..ab6b8d921d6bc 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
@@ -14,15 +14,15 @@ define void @truncate_to_minimal_bitwidths_widen_cast_recipe(ptr %src) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 9, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv1i8.nxv1p0(<vscale x 1 x i8> zeroinitializer, <vscale x 1 x ptr> align 1 splat (ptr null), <vscale x 1 x i1> splat (i1 true), i32 [[TMP7]])
-; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP7]] to i64
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP1]]
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> zeroinitializer, <vscale x 2 x ptr> align 1 splat (ptr null), <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT1:.*]]
-; CHECK: [[EXIT1]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
index c578e0c26e884..c7584291766a8 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
@@ -549,26 +549,26 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
; SCALABLE-NEXT: [[ENTRY:.*:]]
; SCALABLE-NEXT: br label %[[VECTOR_PH:.*]]
; SCALABLE: [[VECTOR_PH]]:
+; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[B]], i64 0
+; SCALABLE-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; SCALABLE-NEXT: [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; SCALABLE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; SCALABLE: [[VECTOR_BODY]]:
; SCALABLE-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SCALABLE-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; SCALABLE-NEXT: [[TMP8:%.*]] = zext i32 [[TMP7]] to i64
-; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
-; SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; SCALABLE-NEXT: [[TMP3:%.*]] = sub i64 [[TMP8]], 1
-; SCALABLE-NEXT: [[TMP4:%.*]] = extractelement <vscale x 2 x i64> [[VEC_IND]], i64 [[TMP3]]
-; SCALABLE-NEXT: store i64 [[TMP4]], ptr [[B]], align 8
+; SCALABLE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
+; SCALABLE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; SCALABLE-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT1]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; SCALABLE-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
; SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP16]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[TMP10]]
; SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
-; SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
; SCALABLE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; SCALABLE-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; SCALABLE: [[MIDDLE_BLOCK]]:
@@ -620,23 +620,23 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
; TF-SCALABLE-NEXT: [[ENTRY:.*:]]
; TF-SCALABLE-NEXT: br label %[[VECTOR_PH:.*]]
; TF-SCALABLE: [[VECTOR_PH]]:
-; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
-; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; TF-SCALABLE-NEXT: [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[B]], i64 0
+; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
+; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; TF-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; TF-SCALABLE: [[VECTOR_BODY]]:
; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; TF-SCALABLE-NEXT: [[TMP13:%.*]] = zext i32 [[TMP9]] to i64
; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; TF-SCALABLE-NEXT: [[TMP3:%.*]] = sub i64 [[TMP13]], 1
-; TF-SCALABLE-NEXT: [[TMP4:%.*]] = extractelement <vscale x 2 x i64> [[VEC_IND]], i64 [[TMP3]]
-; TF-SCALABLE-NEXT: store i64 [[TMP4]], ptr [[B]], align 8
+; TF-SCALABLE-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
; TF-SCALABLE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
+; TF-SCALABLE-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT3]], ptr align 8 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
; TF-SCALABLE-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
; TF-SCALABLE-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; TF-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
index 0ecb9f53eb3fc..831e9b0f3bb4a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
@@ -18,15 +18,15 @@ define void @vf_will_not_generate_any_vector_insts(ptr %src, ptr %dst) {
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4, !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv1i32.nxv1p0(<vscale x 1 x i32> [[BROADCAST_SPLAT3]], <vscale x 1 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> [[BROADCAST_SPLAT3]], <vscale x 2 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP5]] to i64
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
>From d7afd32c3da03789aec85ee46e2c768bed42437d Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 10 Aug 2026 22:10:45 -0700
Subject: [PATCH 6/6] Address comments.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 9 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 44 ++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 23 +-
.../Transforms/Vectorize/VPlanTransforms.h | 3 +-
.../Transforms/LoopVectorize/RISCV/divrem.ll | 37 ++-
.../RISCV/gather-scatter-cost.ll | 232 +++++++++++-------
.../RISCV/strided-access-wide-stride.ll | 6 +-
.../LoopVectorize/RISCV/strided-accesses.ll | 6 +-
.../vf-will-not-generate-any-vector-insts.ll | 12 +-
10 files changed, 223 insertions(+), 151 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 911a4e621a234..69826672bb773 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6892,7 +6892,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
Range.End = Range.Start * 2;
RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
- Config.getMinimalBitwidths());
+ Config.getMinimalBitwidths(), Range);
RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *Plan);
RUN_VPLAN_PASS(VPlanTransforms::narrowScatters, *Plan, CostCtx, Range,
CM.foldTailWithEVL());
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1b8b6bedf6b77..0ff7240af44f5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1527,11 +1527,10 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Compute the cost for a recipe with \p VF using \p Opcode, \p ValTy.
/// This function may not be as accurate as
/// `getCostForRecipeWithOpcode` since it only provides type-based queries.
- static InstructionCost getCostForRecipeWithOpcodeAndTypes(unsigned Opcode,
- Type *ValTy,
- Instruction *I,
- ElementCount VF,
- VPCostContext &Ctx);
+ static InstructionCost
+ getCostForRecipeWithOpcodeAndTypes(unsigned Opcode, Type *ResultTy,
+ ArrayRef<Type *> OpTys, Instruction *I,
+ ElementCount VF, VPCostContext &Ctx);
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b8118ed558f87..77a20dba43e6c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1115,19 +1115,18 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
InstructionCost VPInstruction::getCostForRecipeWithOpcodeAndTypes(
- unsigned Opcode, Type *ValTy, Instruction *I, ElementCount VF,
- VPCostContext &Ctx) {
+ unsigned Opcode, Type *ResultTy, ArrayRef<Type *> OpTys, Instruction *I,
+ ElementCount VF, VPCostContext &Ctx) {
switch (Opcode) {
case VPInstruction::LastActiveLane: {
// LastActiveLane computes the index of the last active lane in a
// predicate mask: NOT + cttz_elts + SUB.
if (VF.isScalar())
- return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ValTy,
- CmpInst::makeCmpResultType(ValTy),
+ return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, OpTys[0],
+ CmpInst::makeCmpResultType(OpTys[0]),
CmpInst::ICMP_EQ, Ctx.CostKind);
- auto *PredTy = toVectorTy(ValTy, VF);
- IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts,
- Type::getInt64Ty(Ctx.LLVMCtx),
+ auto *PredTy = toVectorTy(OpTys[0], VF);
+ IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts, ResultTy,
{PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
// Add cost of NOT operation on the predicate.
@@ -1137,10 +1136,11 @@ InstructionCost VPInstruction::getCostForRecipeWithOpcodeAndTypes(
{TargetTransformInfo::OK_UniformConstantValue,
TargetTransformInfo::OP_None});
// Add cost of SUB operation on the index.
- Cost += Ctx.TTI.getArithmeticInstrCost(
- Instruction::Sub, Type::getInt64Ty(Ctx.LLVMCtx), Ctx.CostKind);
+ Cost += Ctx.TTI.getArithmeticInstrCost(Instruction::Sub, ResultTy,
+ Ctx.CostKind);
return Cost;
}
+ case Instruction::ExtractElement:
case VPInstruction::ExtractLane: {
// Compute the cost of ExtractLane for a vector with the given scalar
// element type. ExtractLane extracts an element at a runtime-determined
@@ -1148,18 +1148,24 @@ InstructionCost VPInstruction::getCostForRecipeWithOpcodeAndTypes(
if (VF.isScalar())
return 0;
return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement,
- toVectorTy(ValTy, VF), Ctx.CostKind);
+ toVectorTy(ResultTy, VF), Ctx.CostKind);
}
case Instruction::Store: {
assert(I && "Querying cost for store must have underlying instruction.");
- return Ctx.TTI.getMemoryOpCost(Opcode, ValTy, getLoadStoreAlignment(I),
- getLoadStoreAddressSpace(I), Ctx.CostKind,
- TTI::getOperandInfo(I->getOperand(0)), I);
+ return Ctx.TTI.getMemoryOpCost(
+ Opcode, OpTys[0], getLoadStoreAlignment(I),
+ cast<PointerType>(OpTys[1])->getAddressSpace(), Ctx.CostKind,
+ TTI::getOperandInfo(I->getOperand(0)), I);
}
case Instruction::Sub: {
- return Ctx.TTI.getArithmeticInstrCost(Opcode, toVectorTy(ValTy, VF),
+ return Ctx.TTI.getArithmeticInstrCost(Opcode, toVectorTy(ResultTy, VF),
Ctx.CostKind);
}
+ case Instruction::ZExt: {
+ return Ctx.TTI.getCastInstrCost(Opcode, toVectorTy(ResultTy, VF),
+ toVectorTy(OpTys[0], VF),
+ TTI::CastContextHint::None, Ctx.CostKind);
+ }
default:
llvm_unreachable("Unsupported opcode");
}
@@ -1400,8 +1406,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
}
case Instruction::ExtractElement:
case VPInstruction::ExtractLane: {
- return getCostForRecipeWithOpcodeAndTypes(
- getOpcode(), getOperand(0)->getScalarType(), nullptr, VF, Ctx);
+ return getCostForRecipeWithOpcodeAndTypes(getOpcode(), getScalarType(), {},
+ nullptr, VF, Ctx);
}
case VPInstruction::AnyOf: {
auto *VecTy = toVectorTy(this->getScalarType(), VF);
@@ -1422,9 +1428,9 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
}
case VPInstruction::LastActiveLane: {
- return getCostForRecipeWithOpcodeAndTypes(VPInstruction::LastActiveLane,
- getOperand(0)->getScalarType(),
- nullptr, VF, Ctx);
+ return getCostForRecipeWithOpcodeAndTypes(
+ VPInstruction::LastActiveLane, getScalarType(),
+ {getOperand(0)->getScalarType()}, nullptr, VF, Ctx);
}
case VPInstruction::ExtractLastActive: {
Type *ScalarTy = this->getScalarType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9f8a1cc149713..3379f5cffe43f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1758,6 +1758,8 @@ void VPlanTransforms::narrowScatters(VPlan &Plan, VPCostContext &Ctx,
[&](ElementCount VF) {
InstructionCost ScatterCost = WidenStoreR->computeCost(VF, Ctx);
Type *ValTy = WidenStoreR->getStoredValue()->getScalarType();
+ Type *AddrTy = WidenStoreR->getAddr()->getScalarType();
+ Type *IndexTy = Plan.getDataLayout().getIndexType(AddrTy);
// ScalarCost = LastActiveLaneCost + ExtractLaneCost +
// ScalarStoreCost.
@@ -1769,15 +1771,19 @@ void VPlanTransforms::narrowScatters(VPlan &Plan, VPCostContext &Ctx,
FoldTailWithEVL
? VPInstruction::getCostForRecipeWithOpcodeAndTypes(
Instruction::Sub, Type::getInt32Ty(Ctx.LLVMCtx),
- nullptr, ElementCount::getFixed(1), Ctx)
+ {}, nullptr, ElementCount::getFixed(1), Ctx) +
+ VPInstruction::getCostForRecipeWithOpcodeAndTypes(
+ Instruction::ZExt, IndexTy,
+ {Type::getInt32Ty(Ctx.LLVMCtx)}, nullptr, VF,
+ Ctx)
: VPInstruction::getCostForRecipeWithOpcodeAndTypes(
- VPInstruction::LastActiveLane,
- Type::getInt1Ty(Ctx.LLVMCtx), nullptr, VF, Ctx);
+ VPInstruction::LastActiveLane, IndexTy,
+ {Type::getInt1Ty(Ctx.LLVMCtx)}, nullptr, VF, Ctx);
ScalarCost += VPInstruction::getCostForRecipeWithOpcodeAndTypes(
- VPInstruction::ExtractLane, ValTy, nullptr, VF, Ctx);
+ VPInstruction::ExtractLane, ValTy, {}, nullptr, VF, Ctx);
ScalarCost += VPInstruction::getCostForRecipeWithOpcodeAndTypes(
- Instruction::Store, ValTy, &WidenStoreR->getIngredient(),
- VF, Ctx);
+ Instruction::Store, nullptr, {ValTy, AddrTy},
+ &WidenStoreR->getIngredient(), VF, Ctx);
return ScalarCost <= ScatterCost;
},
@@ -2533,8 +2539,9 @@ static void licm(VPlan &Plan) {
}
void VPlanTransforms::truncateToMinimalBitwidths(
- VPlan &Plan, const MapVector<Instruction *, uint64_t> &MinBWs) {
- if (Plan.hasScalarVFOnly())
+ VPlan &Plan, const MapVector<Instruction *, uint64_t> &MinBWs,
+ VFRange &Range) {
+ if (Range.Start.isScalar())
return;
// Keep track of created truncates, so they can be re-used. Note that we
// cannot use RAUW after creating a new truncate, as this would could make
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 6a74fa9577d08..56895fa8c8cbf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -303,7 +303,8 @@ struct VPlanTransforms {
/// will be folded later.
static void
truncateToMinimalBitwidths(VPlan &Plan,
- const MapVector<Instruction *, uint64_t> &MinBWs);
+ const MapVector<Instruction *, uint64_t> &MinBWs,
+ VFRange &Range);
/// Replace symbolic strides from \p StridesMap in \p Plan with constants when
/// possible.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index f85278d203bbf..d02dc36f7a6ae 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -631,7 +631,7 @@ for.end:
define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-LABEL: @udiv_sdiv_with_invariant_divisors(
; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
@@ -640,31 +640,30 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[Y:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[P:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.stepvector.nxv4i8()
; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i8> splat (i8 -12), [[TMP1]]
-; CHECK-NEXT: br label [[EXIT:%.*]]
+; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[INDUCTION]], [[LOOP_LATCH]] ], [ [[VEC_IND_NEXT:%.*]], [[EXIT]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 12, [[LOOP_LATCH]] ], [ [[AVL_NEXT:%.*]], [[EXIT]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[INDUCTION]], [[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP_LATCH]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 12, [[VECTOR_BODY]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i8> @llvm.vp.udiv.nxv4i8(<vscale x 4 x i8> [[VEC_IND]], <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
+; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[TMP5]] to <vscale x 4 x i16>
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vp.sdiv.nxv4i16(<vscale x 4 x i16> [[TMP6]], <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
+; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i16> [[TMP8]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP9]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT6]]
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT8]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP_LATCH]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i8> @llvm.vp.udiv.nxv4i8(<vscale x 4 x i8> [[VEC_IND]], <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
-; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 4 x i8> [[TMP4]] to <vscale x 4 x i16>
-; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i16> @llvm.vp.sdiv.nxv4i16(<vscale x 4 x i16> [[TMP5]], <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
-; CHECK-NEXT: [[TMP7:%.*]] = sext <vscale x 4 x i16> [[TMP6]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP7]]
-; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP2]] to i64
-; CHECK-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 1
-; CHECK-NEXT: [[MERGE:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI]], i64 [[TMP9]]
-; CHECK-NEXT: store i32 [[MERGE]], ptr [[P:%.*]], align 4
-; CHECK-NEXT: br label [[EXIT1:%.*]]
+; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
index ccbca35b6f624..e06e7fce3a56c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
@@ -3,69 +3,132 @@
; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+rva23u64,+zvl1024b -S | FileCheck %s -check-prefixes=CHECK,RVA23ZVL1024B
define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
-; CHECK-LABEL: @predicated_uniform_load(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[N:%.*]] to i64
-; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1
-; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 0)
-; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[SMAX2]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = add nuw i32 [[TMP2]], 1
-; CHECK-NEXT: br label [[VECTOR_SCEVCHECK:%.*]]
-; CHECK: vector.scevcheck:
-; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[N]] to i64
-; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 1
-; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP5]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[SMAX]] to i32
-; CHECK-NEXT: [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[SMAX]], 4294967295
-; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT: br i1 [[TMP9]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK: vector.memcheck:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 4
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 4
-; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[COND:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[SRC]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP10]], i32 [[TMP11]]), !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP11]]
-; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[COND]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
-; CHECK-NEXT: [[TMP13:%.*]] = sub i64 [[TMP12]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI]], i64 [[TMP13]]
-; CHECK-NEXT: store i32 [[TMP14]], ptr [[DST]], align 4, !alias.scope [[META6:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT: br label [[EXIT:%.*]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: br label [[LOOP:%.*]]
-; CHECK: loop:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
-; CHECK-NEXT: br i1 [[COND]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]]
-; CHECK: loop.then:
-; CHECK-NEXT: br label [[LOOP_LATCH]]
-; CHECK: loop.else:
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[SRC]], align 4
-; CHECK-NEXT: br label [[LOOP_LATCH]]
-; CHECK: loop.latch:
-; CHECK-NEXT: [[STORE:%.*]] = phi i32 [ [[TMP16]], [[LOOP_ELSE]] ], [ 0, [[LOOP_THEN]] ]
-; CHECK-NEXT: store i32 [[STORE]], ptr [[DST]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp sgt i32 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: exit:
-; CHECK-NEXT: ret void
+; RVA23-LABEL: @predicated_uniform_load(
+; RVA23-NEXT: entry:
+; RVA23-NEXT: [[TMP0:%.*]] = sext i32 [[N:%.*]] to i64
+; RVA23-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1
+; RVA23-NEXT: [[TMP2:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 0)
+; RVA23-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; RVA23-NEXT: [[TMP4:%.*]] = add nuw i32 [[TMP3]], 1
+; RVA23-NEXT: br label [[VECTOR_SCEVCHECK:%.*]]
+; RVA23: vector.scevcheck:
+; RVA23-NEXT: [[TMP5:%.*]] = sext i32 [[N]] to i64
+; RVA23-NEXT: [[TMP6:%.*]] = add nsw i64 [[TMP5]], 1
+; RVA23-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 0)
+; RVA23-NEXT: [[TMP7:%.*]] = trunc i64 [[SMAX]] to i32
+; RVA23-NEXT: [[TMP8:%.*]] = icmp slt i32 [[TMP7]], 0
+; RVA23-NEXT: [[TMP9:%.*]] = icmp ugt i64 [[SMAX]], 4294967295
+; RVA23-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
+; RVA23-NEXT: br i1 [[TMP10]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; RVA23: vector.memcheck:
+; RVA23-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 4
+; RVA23-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 4
+; RVA23-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
+; RVA23-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; RVA23-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; RVA23-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; RVA23: vector.ph:
+; RVA23-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[COND:%.*]], i64 0
+; RVA23-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; RVA23-NEXT: [[TMP11:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; RVA23-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[SRC]], i64 0
+; RVA23-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; RVA23-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST]], i64 0
+; RVA23-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; RVA23-NEXT: br label [[VECTOR_BODY:%.*]]
+; RVA23: vector.body:
+; RVA23-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP4]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RVA23-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; RVA23-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT3]], <vscale x 4 x i1> [[TMP11]], i32 [[TMP12]]), !alias.scope [[META0:![0-9]+]]
+; RVA23-NEXT: [[PREDPHI:%.*]] = select i1 [[COND]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]]
+; RVA23-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; RVA23-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP12]]
+; RVA23-NEXT: [[TMP13:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; RVA23-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; RVA23: middle.block:
+; RVA23-NEXT: br label [[EXIT:%.*]]
+; RVA23: scalar.ph:
+; RVA23-NEXT: br label [[LOOP:%.*]]
+; RVA23: loop:
+; RVA23-NEXT: [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; RVA23-NEXT: br i1 [[COND]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]]
+; RVA23: loop.then:
+; RVA23-NEXT: br label [[LOOP_LATCH]]
+; RVA23: loop.else:
+; RVA23-NEXT: [[TMP14:%.*]] = load i32, ptr [[SRC]], align 4
+; RVA23-NEXT: br label [[LOOP_LATCH]]
+; RVA23: loop.latch:
+; RVA23-NEXT: [[STORE:%.*]] = phi i32 [ [[TMP14]], [[LOOP_ELSE]] ], [ 0, [[LOOP_THEN]] ]
+; RVA23-NEXT: store i32 [[STORE]], ptr [[DST]], align 4
+; RVA23-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; RVA23-NEXT: [[EXITCOND:%.*]] = icmp sgt i32 [[IV]], [[N]]
+; RVA23-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; RVA23: exit:
+; RVA23-NEXT: ret void
+;
+; RVA23ZVL1024B-LABEL: @predicated_uniform_load(
+; RVA23ZVL1024B-NEXT: entry:
+; RVA23ZVL1024B-NEXT: [[TMP0:%.*]] = sext i32 [[N:%.*]] to i64
+; RVA23ZVL1024B-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1
+; RVA23ZVL1024B-NEXT: [[TMP2:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 0)
+; RVA23ZVL1024B-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; RVA23ZVL1024B-NEXT: [[TMP4:%.*]] = add nuw i32 [[TMP3]], 1
+; RVA23ZVL1024B-NEXT: br label [[VECTOR_SCEVCHECK:%.*]]
+; RVA23ZVL1024B: vector.scevcheck:
+; RVA23ZVL1024B-NEXT: [[TMP5:%.*]] = sext i32 [[N]] to i64
+; RVA23ZVL1024B-NEXT: [[TMP6:%.*]] = add nsw i64 [[TMP5]], 1
+; RVA23ZVL1024B-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 0)
+; RVA23ZVL1024B-NEXT: [[TMP7:%.*]] = trunc i64 [[SMAX]] to i32
+; RVA23ZVL1024B-NEXT: [[TMP8:%.*]] = icmp slt i32 [[TMP7]], 0
+; RVA23ZVL1024B-NEXT: [[TMP9:%.*]] = icmp ugt i64 [[SMAX]], 4294967295
+; RVA23ZVL1024B-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
+; RVA23ZVL1024B-NEXT: br i1 [[TMP10]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; RVA23ZVL1024B: vector.memcheck:
+; RVA23ZVL1024B-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 4
+; RVA23ZVL1024B-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 4
+; RVA23ZVL1024B-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
+; RVA23ZVL1024B-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; RVA23ZVL1024B-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; RVA23ZVL1024B-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; RVA23ZVL1024B: vector.ph:
+; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[COND:%.*]], i64 0
+; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; RVA23ZVL1024B-NEXT: [[TMP11:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[SRC]], i64 0
+; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; RVA23ZVL1024B-NEXT: br label [[VECTOR_BODY:%.*]]
+; RVA23ZVL1024B: vector.body:
+; RVA23ZVL1024B-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP4]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RVA23ZVL1024B-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; RVA23ZVL1024B-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT3]], <vscale x 4 x i1> [[TMP11]], i32 [[TMP12]]), !alias.scope [[META0:![0-9]+]]
+; RVA23ZVL1024B-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP12]]
+; RVA23ZVL1024B-NEXT: [[TMP13:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; RVA23ZVL1024B-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; RVA23ZVL1024B: middle.block:
+; RVA23ZVL1024B-NEXT: [[PREDPHI:%.*]] = select i1 [[COND]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]]
+; RVA23ZVL1024B-NEXT: [[TMP14:%.*]] = zext i32 [[TMP12]] to i64
+; RVA23ZVL1024B-NEXT: [[TMP15:%.*]] = sub i64 [[TMP14]], 1
+; RVA23ZVL1024B-NEXT: [[TMP16:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI]], i64 [[TMP15]]
+; RVA23ZVL1024B-NEXT: store i32 [[TMP16]], ptr [[DST]], align 4, !alias.scope [[META6:![0-9]+]], !noalias [[META0]]
+; RVA23ZVL1024B-NEXT: br label [[EXIT:%.*]]
+; RVA23ZVL1024B: scalar.ph:
+; RVA23ZVL1024B-NEXT: br label [[LOOP:%.*]]
+; RVA23ZVL1024B: loop:
+; RVA23ZVL1024B-NEXT: [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; RVA23ZVL1024B-NEXT: br i1 [[COND]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]]
+; RVA23ZVL1024B: loop.then:
+; RVA23ZVL1024B-NEXT: br label [[LOOP_LATCH]]
+; RVA23ZVL1024B: loop.else:
+; RVA23ZVL1024B-NEXT: [[TMP17:%.*]] = load i32, ptr [[SRC]], align 4
+; RVA23ZVL1024B-NEXT: br label [[LOOP_LATCH]]
+; RVA23ZVL1024B: loop.latch:
+; RVA23ZVL1024B-NEXT: [[STORE:%.*]] = phi i32 [ [[TMP17]], [[LOOP_ELSE]] ], [ 0, [[LOOP_THEN]] ]
+; RVA23ZVL1024B-NEXT: store i32 [[STORE]], ptr [[DST]], align 4
+; RVA23ZVL1024B-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; RVA23ZVL1024B-NEXT: [[EXITCOND:%.*]] = icmp sgt i32 [[IV]], [[N]]
+; RVA23ZVL1024B-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; RVA23ZVL1024B: exit:
+; RVA23ZVL1024B-NEXT: ret void
;
entry:
br label %loop
@@ -140,32 +203,32 @@ define void @store_to_addr_generated_from_invariant_addr(ptr noalias %p0, ptr no
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[P0:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[P0:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P1:%.*]], <vscale x 2 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2p0.nxv2p0(<vscale x 2 x ptr> [[BROADCAST_SPLAT]], <vscale x 2 x ptr> align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
-; CHECK-NEXT: [[TMP5:%.*]] = load i64, ptr [[P2:%.*]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P3:%.*]], i64 [[TMP5]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[TMP6]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> zeroinitializer, <vscale x 2 x ptr> align 1 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP4]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[P1:%.*]], <vscale x 2 x i64> [[VEC_IND]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2p0.nxv2p0(<vscale x 2 x ptr> [[BROADCAST_SPLAT1]], <vscale x 2 x ptr> align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = load i64, ptr [[P2:%.*]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P3:%.*]], i64 [[TMP6]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> align 4 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> align 4 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> zeroinitializer, <vscale x 2 x ptr> align 1 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: br label [[EXIT:%.*]]
+; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
@@ -263,6 +326,3 @@ exit:
}
attributes #0 = { "target-features"="+zve64x,+zvl256b" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; RVA23: {{.*}}
-; RVA23ZVL1024B: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
index 8dd170a99f143..da9dce90f33a0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
@@ -24,14 +24,14 @@ define void @stride_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst,
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP4]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[INDEX]] to i64
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP7]], 3000000000
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP11]]
-; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP8]], i64 3000000000, <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP8]], i64 3000000000, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
; CHECK-NEXT: [[TMP15:%.*]] = zext i32 [[TMP6]] to i64
; CHECK-NEXT: [[TMP16:%.*]] = sub i64 [[TMP15]], 1
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <vscale x 8 x i8> [[TMP14]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <vscale x 16 x i8> [[TMP14]], i64 [[TMP16]]
; CHECK-NEXT: store i8 [[TMP17]], ptr [[DST]], align 1
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP6]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 5dcacebd897fb..2836c0906bd5d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -2156,13 +2156,13 @@ define void @iv_start_is_addrec(ptr %p, ptr noalias %q) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 129, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[INDEX]], -8
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP4]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP6]], i64 -8, <vscale x 8 x i1> splat (i1 true), i32 [[TMP2]])
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP6]], i64 -8, <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP3]], 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <vscale x 8 x i8> [[WIDE_MASKED_GATHER]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <vscale x 16 x i8> [[TMP9]], i64 [[TMP8]]
; CHECK-NEXT: store i8 [[TMP7]], ptr [[Q]], align 1
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP3]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
index 831e9b0f3bb4a..dca4f47738309 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
@@ -18,15 +18,15 @@ define void @vf_will_not_generate_any_vector_insts(ptr %src, ptr %dst) {
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4, !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> [[BROADCAST_SPLAT3]], <vscale x 2 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT3]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP5]] to i64
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
More information about the llvm-commits
mailing list