[llvm] [RISCV] Enable shouldMaximizeVectorBandwidth (PR #216005)
Pengcheng Wang via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 00:11:56 PDT 2026
https://github.com/wangpc-pp updated https://github.com/llvm/llvm-project/pull/216005
>From 2812e4608ad5fad9a90fc3a85e2effc371008c13 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Tue, 11 Aug 2026 19:52:03 +0800
Subject: [PATCH 1/3] [RISCV] Enable shouldMaximizeVectorBandwidth
In this PR, we size vector factors by the smallest type in the loop
rather than the widest, matching AArch64's behaviour. This lets
loops that mix narrow and wide types use the full vector register
bandwidth instead of being limited by the widest type.
Overly wide vector factors are pruned by register pressure estimation,
which RISC-V already opts into via shouldConsiderVectorizationRegPressure,
so this does not force excessively wide factors.
Assisted-by: TRAE CLI (Opus 4.8)
---
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 9 +
.../Target/RISCV/RISCVTargetTransformInfo.h | 3 +
.../Transforms/LoopVectorize/RISCV/bf16.ll | 16 +-
.../RISCV/blend-any-of-reduction-cost.ll | 28 +-
.../LoopVectorize/RISCV/dead-ops-cost.ll | 70 ++-
.../RISCV/gather-scatter-cost.ll | 142 ++++---
.../LoopVectorize/RISCV/illegal-type.ll | 22 +-
.../LoopVectorize/RISCV/induction-costs.ll | 28 +-
.../LoopVectorize/RISCV/inloop-reduction.ll | 28 +-
.../RISCV/masked_gather_scatter.ll | 30 +-
.../RISCV/partial-reduce-dot-product.ll | 304 ++++++-------
.../RISCV/partial-reduce-with-predicate.ll | 400 ++++++++----------
.../LoopVectorize/RISCV/partial-reduce.ll | 92 ++--
.../LoopVectorize/RISCV/pointer-induction.ll | 32 +-
.../RISCV/tail-folding-cast-intrinsics.ll | 86 ++--
.../truncate-to-minimal-bitwidth-cost.ll | 46 +-
.../truncate-to-minimal-bitwidth-evl-crash.ll | 12 +-
.../RISCV/type-info-cache-evl-crash.ll | 22 +-
18 files changed, 706 insertions(+), 664 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index cd14ff3d2dded..62ba4e92be257 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -479,6 +479,15 @@ RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
llvm_unreachable("Unsupported register kind");
}
+bool RISCVTTIImpl::shouldMaximizeVectorBandwidth(
+ TargetTransformInfo::RegisterKind K) const {
+ // Size vector factors by the smallest type in the loop rather than the
+ // widest, so loops mixing narrow and wide types can use the full vector
+ // register bandwidth. Overly wide factors are pruned separately by register
+ // pressure estimation (shouldConsiderVectorizationRegPressure).
+ return K != TargetTransformInfo::RGK_Scalar && ST->hasVInstructions();
+}
+
InstructionCost RISCVTTIImpl::getStaticDataAddrGenerationCost(
const TTI::TargetCostKind CostKind) const {
switch (CostKind) {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 447c3d7c3326e..67ea98c3f5a72 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -134,6 +134,9 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
TypeSize
getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override;
+ bool shouldMaximizeVectorBandwidth(
+ TargetTransformInfo::RegisterKind K) const override;
+
unsigned getRegUsageForType(Type *Ty) const override;
unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const override;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
index 9951165b77da6..c1c0acab2afaf 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
@@ -124,17 +124,17 @@ define void @vfwmaccbf16.vv(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64
; ZVFBFMIN: [[VECTOR_BODY]]:
; ZVFBFMIN-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ZVFBFMIN-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ZVFBFMIN-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; ZVFBFMIN-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
; ZVFBFMIN-NEXT: [[TMP1:%.*]] = getelementptr bfloat, ptr [[A]], i64 [[INDEX]]
; ZVFBFMIN-NEXT: [[TMP2:%.*]] = getelementptr bfloat, ptr [[B]], i64 [[INDEX]]
; ZVFBFMIN-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[C]], i64 [[INDEX]]
-; ZVFBFMIN-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x bfloat> @llvm.vp.load.nxv4bf16.p0(ptr align 2 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; ZVFBFMIN-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x bfloat> @llvm.vp.load.nxv4bf16.p0(ptr align 2 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; ZVFBFMIN-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; ZVFBFMIN-NEXT: [[TMP4:%.*]] = fpext <vscale x 4 x bfloat> [[VP_OP_LOAD]] to <vscale x 4 x float>
-; ZVFBFMIN-NEXT: [[TMP5:%.*]] = fpext <vscale x 4 x bfloat> [[VP_OP_LOAD1]] to <vscale x 4 x float>
-; ZVFBFMIN-NEXT: [[TMP6:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], <vscale x 4 x float> [[VP_OP_LOAD2]])
-; ZVFBFMIN-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP6]], ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVFBFMIN-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 8 x bfloat> @llvm.vp.load.nxv8bf16.p0(ptr align 2 [[TMP1]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVFBFMIN-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 8 x bfloat> @llvm.vp.load.nxv8bf16.p0(ptr align 2 [[TMP2]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVFBFMIN-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 8 x float> @llvm.vp.load.nxv8f32.p0(ptr align 4 [[TMP3]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVFBFMIN-NEXT: [[TMP4:%.*]] = fpext <vscale x 8 x bfloat> [[VP_OP_LOAD]] to <vscale x 8 x float>
+; ZVFBFMIN-NEXT: [[TMP5:%.*]] = fpext <vscale x 8 x bfloat> [[VP_OP_LOAD1]] to <vscale x 8 x float>
+; ZVFBFMIN-NEXT: [[TMP6:%.*]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[TMP4]], <vscale x 8 x float> [[TMP5]], <vscale x 8 x float> [[VP_OP_LOAD2]])
+; ZVFBFMIN-NEXT: call void @llvm.vp.store.nxv8f32.p0(<vscale x 8 x float> [[TMP6]], ptr align 4 [[TMP3]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
; ZVFBFMIN-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
; ZVFBFMIN-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]]
; ZVFBFMIN-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
index 9f7ac7ac1771f..7ccd4f8ac3fcc 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
@@ -64,26 +64,26 @@ define i32 @any_of_reduction_used_in_blend_with_multiple_phis(ptr %src, i64 %N,
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[ANY_OF_RED:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ANY_OF_RED_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH]] ]
-; CHECK-NEXT: br i1 [[C_0]], label %[[X_1:.*]], label %[[ELSE_1:.*]]
-; CHECK: [[ELSE_1]]:
-; CHECK-NEXT: br i1 [[C_1]], label %[[X_1]], label %[[ELSE_2:.*]]
-; CHECK: [[ELSE_2]]:
+; CHECK-NEXT: [[ANY_OF_RED:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ANY_OF_RED_NEXT:%.*]], %[[EXIT:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: br i1 [[C_0]], label %[[X_1:.*]], label %[[LOOP_LATCH:.*]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: br i1 [[C_1]], label %[[X_1]], label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[SRC]], align 8
; CHECK-NEXT: [[C_2:%.*]] = icmp eq ptr [[L]], null
; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C_2]], i32 0, i32 [[ANY_OF_RED]]
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[X_1]]:
-; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[ANY_OF_RED]], %[[LOOP_HEADER]] ], [ [[ANY_OF_RED]], %[[ELSE_1]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[ANY_OF_RED_NEXT]] = phi i32 [ [[P]], %[[X_1]] ], [ [[SEL]], %[[ELSE_2]] ]
+; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[ANY_OF_RED]], %[[LOOP_HEADER]] ], [ [[ANY_OF_RED]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[ANY_OF_RED_NEXT]] = phi i32 [ [[P]], %[[X_1]] ], [ [[SEL]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[ANY_OF_RED_NEXT]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1:.*]], label %[[LOOP_HEADER]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[ANY_OF_RED_NEXT]], %[[EXIT]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 23d9b5c5b7c52..478f7126a4413 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -154,15 +154,15 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP2]], 7
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP2]], 15
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i32 8, i32 [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i32 16, i32 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[TMP9]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[C]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE18:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE32:.*]] ]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[B]], i32 [[INDEX]]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
@@ -199,13 +199,53 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16]]
; CHECK: [[PRED_STORE_CONTINUE16]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18]]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
; CHECK: [[PRED_STORE_IF17]]:
; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE18]]
; CHECK: [[PRED_STORE_CONTINUE18]]:
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[TMP11]], <8 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META13]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF18:.*]], label %[[PRED_STORE_CONTINUE19:.*]]
+; CHECK: [[PRED_STORE_IF18]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE19]]
+; CHECK: [[PRED_STORE_CONTINUE19]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
+; CHECK: [[PRED_STORE_IF19]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE20]]
+; CHECK: [[PRED_STORE_CONTINUE20]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
+; CHECK: [[PRED_STORE_IF21]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE22]]
+; CHECK: [[PRED_STORE_CONTINUE22]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
+; CHECK: [[PRED_STORE_IF23]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE24]]
+; CHECK: [[PRED_STORE_CONTINUE24]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]
+; CHECK: [[PRED_STORE_IF25]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE26]]
+; CHECK: [[PRED_STORE_CONTINUE26]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]
+; CHECK: [[PRED_STORE_IF27]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE28]]
+; CHECK: [[PRED_STORE_CONTINUE28]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]
+; CHECK: [[PRED_STORE_IF29]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE30]]
+; CHECK: [[PRED_STORE_CONTINUE30]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32]]
+; CHECK: [[PRED_STORE_IF31]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE32]]
+; CHECK: [[PRED_STORE_CONTINUE32]]:
+; CHECK-NEXT: call void @llvm.masked.store.v16i32.p0(<16 x i32> zeroinitializer, ptr align 4 [[TMP11]], <16 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META13]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -332,20 +372,20 @@ define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %s
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[EVL_BASED_IV]], 1
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 1
; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP11]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP22]], i32 -1
; CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP10]], 2
-; CHECK-NEXT: [[WIDE_VP_LOAD:%.*]] = call <vscale x 8 x i8> @llvm.vp.load.nxv8i8.p0(ptr align 1 [[TMP13]], <vscale x 8 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i8>, <vscale x 4 x i8> } @llvm.vector.deinterleave2.nxv8i8(<vscale x 8 x i8> [[WIDE_VP_LOAD]])
-; CHECK-NEXT: [[TMP17:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 0
-; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 1
-; CHECK-NEXT: [[TMP18:%.*]] = zext <vscale x 4 x i8> [[TMP12]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[WIDE_VP_LOAD:%.*]] = call <vscale x 32 x i8> @llvm.vp.load.nxv32i8.p0(ptr align 1 [[TMP13]], <vscale x 32 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[WIDE_VP_LOAD]])
+; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[TMP12]] to <vscale x 16 x i32>
; CHECK-NEXT: [[TMP14:%.*]] = shl i64 [[EVL_BASED_IV]], 3
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP14]]
-; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64(<vscale x 4 x i32> [[TMP18]], ptr align 4 [[TMP15]], i64 8, <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv16i32.p0.i64(<vscale x 16 x i32> [[TMP17]], ptr align 4 [[TMP15]], i64 8, <vscale x 16 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[TMP16:%.*]] = zext i32 [[TMP10]] to i64
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
index c1ed5b1327073..6eae71d81a91c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
@@ -158,23 +158,23 @@ define void @store_to_addr_generated_from_invariant_addr(ptr noalias %p0, ptr no
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[P0:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x ptr> poison, ptr [[P0:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 16 x ptr> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P1:%.*]], i64 [[TMP2]]
-; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2p0.p0.i64(<vscale x 2 x ptr> [[BROADCAST_SPLAT1]], ptr align 8 [[TMP5]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv16p0.p0.i64(<vscale x 16 x ptr> [[BROADCAST_SPLAT]], ptr align 8 [[TMP5]], i64 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP3]])
; CHECK-NEXT: [[TMP6:%.*]] = load i64, ptr [[P2:%.*]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P3:%.*]], i64 [[TMP6]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> align 4 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> align 4 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> zeroinitializer, <vscale x 2 x ptr> align 1 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x ptr> poison, ptr [[TMP8]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x ptr> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i32.nxv16p0(<vscale x 16 x i32> zeroinitializer, <vscale x 16 x ptr> align 4 [[BROADCAST_SPLAT2]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i32.nxv16p0(<vscale x 16 x i32> zeroinitializer, <vscale x 16 x ptr> align 4 [[BROADCAST_SPLAT2]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x ptr> align 1 [[BROADCAST_SPLAT2]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP3]])
; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
@@ -207,46 +207,87 @@ exit:
; Test for https://github.com/llvm/llvm-project/issues/169948.
define i8 @mixed_gather_scatters(ptr %A, ptr %B, ptr %C) #0 {
-; CHECK-LABEL: @mixed_gather_scatters(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[A:%.*]], align 8
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[B:%.*]], align 8
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 1 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[C:%.*]], align 8
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 1 x ptr> [[BROADCAST_SPLATINSERT4]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i8> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 10, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 1, i1 true)
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 1 x i64> @llvm.vp.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt <vscale x 1 x i64> [[WIDE_MASKED_GATHER]], zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 1 x i1> [[TMP4]] to <vscale x 1 x i8>
-; CHECK-NEXT: [[TMP6:%.*]] = or <vscale x 1 x i8> [[VEC_PHI]], [[TMP5]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 1 x i64> @llvm.vp.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> align 8 [[BROADCAST_SPLAT2]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: [[TMP7:%.*]] = icmp sgt <vscale x 1 x i64> [[WIDE_MASKED_GATHER3]], zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = zext <vscale x 1 x i1> [[TMP7]] to <vscale x 1 x i8>
-; CHECK-NEXT: [[TMP9:%.*]] = or <vscale x 1 x i8> [[TMP6]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = or <vscale x 1 x i8> [[TMP9]], splat (i8 1)
-; CHECK-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 1 x i64> @llvm.vp.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> align 8 [[BROADCAST_SPLAT5]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: [[TMP11:%.*]] = icmp sgt <vscale x 1 x i64> [[WIDE_MASKED_GATHER6]], zeroinitializer
-; CHECK-NEXT: [[TMP12:%.*]] = zext <vscale x 1 x i1> [[TMP11]] to <vscale x 1 x i8>
-; CHECK-NEXT: [[TMP13:%.*]] = or <vscale x 1 x i8> [[TMP10]], [[TMP12]]
-; CHECK-NEXT: [[TMP14]] = call <vscale x 1 x i8> @llvm.vp.merge.nxv1i8(<vscale x 1 x i1> splat (i1 true), <vscale x 1 x i8> [[TMP13]], <vscale x 1 x i8> [[VEC_PHI]], i32 [[TMP3]])
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP3]]
-; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[TMP16:%.*]] = call i8 @llvm.vector.reduce.or.nxv1i8(<vscale x 1 x i8> [[TMP14]])
-; CHECK-NEXT: br label [[EXIT:%.*]]
-; CHECK: exit:
-; CHECK-NEXT: ret i8 [[TMP16]]
+; RVA23-LABEL: @mixed_gather_scatters(
+; RVA23-NEXT: entry:
+; RVA23-NEXT: br label [[VECTOR_PH:%.*]]
+; RVA23: vector.ph:
+; RVA23-NEXT: [[TMP0:%.*]] = load ptr, ptr [[A:%.*]], align 8
+; RVA23-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP0]], i64 0
+; RVA23-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; RVA23-NEXT: [[TMP1:%.*]] = load ptr, ptr [[B:%.*]], align 8
+; RVA23-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP1]], i64 0
+; RVA23-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; RVA23-NEXT: [[TMP2:%.*]] = load ptr, ptr [[C:%.*]], align 8
+; RVA23-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP2]], i64 0
+; RVA23-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; RVA23-NEXT: br label [[VECTOR_BODY:%.*]]
+; RVA23: vector.body:
+; RVA23-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i8> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; RVA23-NEXT: [[AVL:%.*]] = phi i32 [ 10, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RVA23-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; RVA23-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.vp.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP3]])
+; RVA23-NEXT: [[TMP4:%.*]] = icmp sgt <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], zeroinitializer
+; RVA23-NEXT: [[TMP5:%.*]] = zext <vscale x 4 x i1> [[TMP4]] to <vscale x 4 x i8>
+; RVA23-NEXT: [[TMP6:%.*]] = or <vscale x 4 x i8> [[VEC_PHI]], [[TMP5]]
+; RVA23-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i64> @llvm.vp.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[BROADCAST_SPLAT2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP3]])
+; RVA23-NEXT: [[TMP7:%.*]] = icmp sgt <vscale x 4 x i64> [[WIDE_MASKED_GATHER3]], zeroinitializer
+; RVA23-NEXT: [[TMP8:%.*]] = zext <vscale x 4 x i1> [[TMP7]] to <vscale x 4 x i8>
+; RVA23-NEXT: [[TMP9:%.*]] = or <vscale x 4 x i8> [[TMP6]], [[TMP8]]
+; RVA23-NEXT: [[TMP10:%.*]] = or <vscale x 4 x i8> [[TMP9]], splat (i8 1)
+; RVA23-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 4 x i64> @llvm.vp.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[BROADCAST_SPLAT5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP3]])
+; RVA23-NEXT: [[TMP11:%.*]] = icmp sgt <vscale x 4 x i64> [[WIDE_MASKED_GATHER6]], zeroinitializer
+; RVA23-NEXT: [[TMP12:%.*]] = zext <vscale x 4 x i1> [[TMP11]] to <vscale x 4 x i8>
+; RVA23-NEXT: [[TMP13:%.*]] = or <vscale x 4 x i8> [[TMP10]], [[TMP12]]
+; RVA23-NEXT: [[TMP14]] = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i8> [[TMP13]], <vscale x 4 x i8> [[VEC_PHI]], i32 [[TMP3]])
+; RVA23-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP3]]
+; RVA23-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; RVA23-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; RVA23: middle.block:
+; RVA23-NEXT: [[TMP16:%.*]] = call i8 @llvm.vector.reduce.or.nxv4i8(<vscale x 4 x i8> [[TMP14]])
+; RVA23-NEXT: br label [[EXIT:%.*]]
+; RVA23: exit:
+; RVA23-NEXT: ret i8 [[TMP16]]
+;
+; RVA23ZVL1024B-LABEL: @mixed_gather_scatters(
+; RVA23ZVL1024B-NEXT: entry:
+; RVA23ZVL1024B-NEXT: br label [[VECTOR_PH:%.*]]
+; RVA23ZVL1024B: vector.ph:
+; RVA23ZVL1024B-NEXT: [[TMP0:%.*]] = load ptr, ptr [[A:%.*]], align 8
+; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[TMP0]], i64 0
+; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
+; RVA23ZVL1024B-NEXT: [[TMP1:%.*]] = load ptr, ptr [[B:%.*]], align 8
+; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[TMP1]], i64 0
+; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 1 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
+; RVA23ZVL1024B-NEXT: [[TMP2:%.*]] = load ptr, ptr [[C:%.*]], align 8
+; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[TMP2]], i64 0
+; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 1 x ptr> [[BROADCAST_SPLATINSERT4]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
+; RVA23ZVL1024B-NEXT: br label [[VECTOR_BODY:%.*]]
+; RVA23ZVL1024B: vector.body:
+; RVA23ZVL1024B-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i8> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; RVA23ZVL1024B-NEXT: [[AVL:%.*]] = phi i32 [ 10, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RVA23ZVL1024B-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 1, i1 true)
+; RVA23ZVL1024B-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 1 x i64> @llvm.vp.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP3]])
+; RVA23ZVL1024B-NEXT: [[TMP4:%.*]] = icmp sgt <vscale x 1 x i64> [[WIDE_MASKED_GATHER]], zeroinitializer
+; RVA23ZVL1024B-NEXT: [[TMP5:%.*]] = zext <vscale x 1 x i1> [[TMP4]] to <vscale x 1 x i8>
+; RVA23ZVL1024B-NEXT: [[TMP6:%.*]] = or <vscale x 1 x i8> [[VEC_PHI]], [[TMP5]]
+; RVA23ZVL1024B-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 1 x i64> @llvm.vp.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> align 8 [[BROADCAST_SPLAT2]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP3]])
+; RVA23ZVL1024B-NEXT: [[TMP7:%.*]] = icmp sgt <vscale x 1 x i64> [[WIDE_MASKED_GATHER3]], zeroinitializer
+; RVA23ZVL1024B-NEXT: [[TMP8:%.*]] = zext <vscale x 1 x i1> [[TMP7]] to <vscale x 1 x i8>
+; RVA23ZVL1024B-NEXT: [[TMP9:%.*]] = or <vscale x 1 x i8> [[TMP6]], [[TMP8]]
+; RVA23ZVL1024B-NEXT: [[TMP10:%.*]] = or <vscale x 1 x i8> [[TMP9]], splat (i8 1)
+; RVA23ZVL1024B-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 1 x i64> @llvm.vp.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> align 8 [[BROADCAST_SPLAT5]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP3]])
+; RVA23ZVL1024B-NEXT: [[TMP11:%.*]] = icmp sgt <vscale x 1 x i64> [[WIDE_MASKED_GATHER6]], zeroinitializer
+; RVA23ZVL1024B-NEXT: [[TMP12:%.*]] = zext <vscale x 1 x i1> [[TMP11]] to <vscale x 1 x i8>
+; RVA23ZVL1024B-NEXT: [[TMP13:%.*]] = or <vscale x 1 x i8> [[TMP10]], [[TMP12]]
+; RVA23ZVL1024B-NEXT: [[TMP14]] = call <vscale x 1 x i8> @llvm.vp.merge.nxv1i8(<vscale x 1 x i1> splat (i1 true), <vscale x 1 x i8> [[TMP13]], <vscale x 1 x i8> [[VEC_PHI]], i32 [[TMP3]])
+; RVA23ZVL1024B-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP3]]
+; RVA23ZVL1024B-NEXT: [[TMP15:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; RVA23ZVL1024B-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; RVA23ZVL1024B: middle.block:
+; RVA23ZVL1024B-NEXT: [[TMP16:%.*]] = call i8 @llvm.vector.reduce.or.nxv1i8(<vscale x 1 x i8> [[TMP14]])
+; RVA23ZVL1024B-NEXT: br label [[EXIT:%.*]]
+; RVA23ZVL1024B: exit:
+; RVA23ZVL1024B-NEXT: ret i8 [[TMP16]]
;
entry:
br label %loop
@@ -279,6 +320,3 @@ exit:
}
attributes #0 = { "target-features"="+zve64x,+zvl256b" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; RVA23: {{.*}}
-; RVA23ZVL1024B: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll b/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll
index 03f404e98d573..93fc95e8967e4 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll
@@ -102,29 +102,29 @@ define void @uniform_store_i1(ptr noalias %dst, ptr noalias %start, i64 %N) {
; CHECK-LABEL: @uniform_store_i1(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 512
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 63
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 511
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[N_VEC]], 3
; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[TMP1]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <32 x ptr> poison, ptr [[START]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <32 x ptr> [[BROADCAST_SPLATINSERT]], <32 x ptr> poison, <32 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <256 x ptr> poison, ptr [[START]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <256 x ptr> [[BROADCAST_SPLATINSERT]], <256 x ptr> poison, <256 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <32 x i64> <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56, i64 64, i64 72, i64 80, i64 88, i64 96, i64 104, i64 112, i64 120, i64 128, i64 136, i64 144, i64 152, i64 160, i64 168, i64 176, i64 184, i64 192, i64 200, i64 208, i64 216, i64 224, i64 232, i64 240, i64 248>
-; CHECK-NEXT: [[STEP_ADD:%.*]] = getelementptr i8, <32 x ptr> [[TMP2]], <32 x i64> splat (i64 256)
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
-; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 512
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <256 x i64> <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56, i64 64, i64 72, i64 80, i64 88, i64 96, i64 104, i64 112, i64 120, i64 128, i64 136, i64 144, i64 152, i64 160, i64 168, i64 176, i64 184, i64 192, i64 200, i64 208, i64 216, i64 224, i64 232, i64 240, i64 248, i64 256, i64 264, i64 272, i64 280, i64 288, i64 296, i64 304, i64 312, i64 320, i64 328, i64 336, i64 344, i64 352, i64 360, i64 368, i64 376, i64 384, i64 392, i64 400, i64 408, i64 416, i64 424, i64 432, i64 440, i64 448, i64 456, i64 464, i64 472, i64 480, i64 488, i64 496, i64 504, i64 512, i64 520, i64 528, i64 536, i64 544, i64 552, i64 560, i64 568, i64 576, i64 584, i64 592, i64 600, i64 608, i64 616, i64 624, i64 632, i64 640, i64 648, i64 656, i64 664, i64 672, i64 680, i64 688, i64 696, i64 704, i64 712, i64 720, i64 728, i64 736, i64 744, i64 752, i64 760, i64 768, i64 776, i64 784, i64 792, i64 800, i64 808, i64 816, i64 824, i64 832, i64 840, i64 848, i64 856, i64 864, i64 872, i64 880, i64 888, i64 896, i64 904, i64 912, i64 920, i64 928, i64 936, i64 944, i64 952, i64 960, i64 968, i64 976, i64 984, i64 992, i64 1000, i64 1008, i64 1016, i64 1024, i64 1032, i64 1040, i64 1048, i64 1056, i64 1064, i64 1072, i64 1080, i64 1088, i64 1096, i64 1104, i64 1112, i64 1120, i64 1128, i64 1136, i64 1144, i64 1152, i64 1160, i64 1168, i64 1176, i64 1184, i64 1192, i64 1200, i64 1208, i64 1216, i64 1224, i64 1232, i64 1240, i64 1248, i64 1256, i64 1264, i64 1272, i64 1280, i64 1288, i64 1296, i64 1304, i64 1312, i64 1320, i64 1328, i64 1336, i64 1344, i64 1352, i64 1360, i64 1368, i64 1376, i64 1384, i64 1392, i64 1400, i64 1408, i64 1416, i64 1424, i64 1432, i64 1440, i64 1448, i64 1456, i64 1464, i64 1472, i64 1480, i64 1488, i64 1496, i64 1504, i64 1512, i64 1520, i64 1528, i64 1536, i64 1544, i64 1552, i64 1560, i64 1568, i64 1576, i64 1584, i64 1592, i64 1600, i64 1608, i64 1616, i64 1624, i64 1632, i64 1640, i64 1648, i64 1656, i64 1664, i64 1672, i64 1680, i64 1688, i64 1696, i64 1704, i64 1712, i64 1720, i64 1728, i64 1736, i64 1744, i64 1752, i64 1760, i64 1768, i64 1776, i64 1784, i64 1792, i64 1800, i64 1808, i64 1816, i64 1824, i64 1832, i64 1840, i64 1848, i64 1856, i64 1864, i64 1872, i64 1880, i64 1888, i64 1896, i64 1904, i64 1912, i64 1920, i64 1928, i64 1936, i64 1944, i64 1952, i64 1960, i64 1968, i64 1976, i64 1984, i64 1992, i64 2000, i64 2008, i64 2016, i64 2024, i64 2032, i64 2040>
+; CHECK-NEXT: [[STEP_ADD:%.*]] = getelementptr i8, <256 x ptr> [[VECTOR_GEP]], <256 x i64> splat (i64 2048)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 512
+; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 4096
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, <32 x ptr> [[STEP_ADD]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <32 x ptr> [[TMP5]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <32 x i1> [[TMP7]], i64 31
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i64, <256 x ptr> [[STEP_ADD]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <256 x ptr> [[WIDE_GEP]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <256 x i1> [[TMP5]], i64 255
; CHECK-NEXT: store i1 [[TMP8]], ptr [[DST:%.*]], align 1
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
index 3a796fe80af11..a9d6b34c106fd 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
@@ -172,29 +172,29 @@ define void @redundant_iv_trunc_for_cse(ptr noalias %src, ptr noalias %dst, i64
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <vscale x 4 x i32> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <vscale x 16 x i32> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP3]] to i64
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP3]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[EVL_BASED_IV]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP3]])
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <vscale x 4 x i32> [[VP_OP_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = shl <vscale x 4 x i32> [[VEC_IND1]], splat (i32 16)
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[VEC_IND]]
-; CHECK-NEXT: [[TMP7:%.*]] = trunc <vscale x 4 x i32> [[PREDPHI]] to <vscale x 4 x i8>
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i32> @llvm.vp.load.nxv16i32.p0(ptr align 4 [[TMP4]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <vscale x 16 x i32> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = shl <vscale x 16 x i32> [[VEC_IND1]], splat (i32 16)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> [[VEC_IND]]
+; CHECK-NEXT: [[TMP7:%.*]] = trunc <vscale x 16 x i32> [[PREDPHI]] to <vscale x 16 x i8>
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[EVL_BASED_IV]]
-; CHECK-NEXT: call void @llvm.vp.store.nxv4i8.p0(<vscale x 4 x i8> [[TMP7]], ptr align 1 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP7]], ptr align 1 [[TMP8]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP3]])
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[VEC_IND_NEXT2]] = add <vscale x 4 x i32> [[VEC_IND1]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT2]] = add <vscale x 16 x i32> [[VEC_IND1]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
index 6b22001dde07b..3145dc5b39957 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
@@ -15,8 +15,8 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
; OUTLOOP-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
; OUTLOOP: for.body.preheader:
; OUTLOOP-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; OUTLOOP-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; OUTLOOP-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 8)
+; OUTLOOP-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 3
+; OUTLOOP-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 16)
; OUTLOOP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], [[UMAX]]
; OUTLOOP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; OUTLOOP: vector.ph:
@@ -25,16 +25,16 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
; OUTLOOP-NEXT: br label [[VECTOR_BODY:%.*]]
; OUTLOOP: vector.body:
; OUTLOOP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; OUTLOOP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; OUTLOOP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
; OUTLOOP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[INDEX]]
-; OUTLOOP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP7]], align 2
-; OUTLOOP-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i32>
-; OUTLOOP-NEXT: [[TMP10]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP9]]
+; OUTLOOP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP7]], align 2
+; OUTLOOP-NEXT: [[TMP4:%.*]] = sext <vscale x 8 x i16> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; OUTLOOP-NEXT: [[TMP6]] = add <vscale x 8 x i32> [[VEC_PHI]], [[TMP4]]
; OUTLOOP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
; OUTLOOP-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; OUTLOOP-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; OUTLOOP: middle.block:
-; OUTLOOP-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP10]])
+; OUTLOOP-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP6]])
; OUTLOOP-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; OUTLOOP-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
; OUTLOOP: scalar.ph:
@@ -117,20 +117,20 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
; IF-EVL-OUTLOOP-NEXT: br label [[VECTOR_BODY:%.*]]
; IF-EVL-OUTLOOP: vector.body:
; IF-EVL-OUTLOOP-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
; IF-EVL-OUTLOOP-NEXT: [[AVL:%.*]] = phi i32 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; IF-EVL-OUTLOOP-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
; IF-EVL-OUTLOOP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[EVL_BASED_IV]]
-; IF-EVL-OUTLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.vp.load.nxv4i16.p0(ptr align 2 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
-; IF-EVL-OUTLOOP-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i16> [[VP_OP_LOAD]] to <vscale x 4 x i32>
-; IF-EVL-OUTLOOP-NEXT: [[VP_OP:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP9]]
-; IF-EVL-OUTLOOP-NEXT: [[TMP10]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[VP_OP]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP5]])
+; IF-EVL-OUTLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP7]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP5]])
+; IF-EVL-OUTLOOP-NEXT: [[TMP2:%.*]] = sext <vscale x 8 x i16> [[VP_OP_LOAD]] to <vscale x 8 x i32>
+; IF-EVL-OUTLOOP-NEXT: [[TMP3:%.*]] = add <vscale x 8 x i32> [[VEC_PHI]], [[TMP2]]
+; IF-EVL-OUTLOOP-NEXT: [[TMP4]] = call <vscale x 8 x i32> @llvm.vp.merge.nxv8i32(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x i32> [[TMP3]], <vscale x 8 x i32> [[VEC_PHI]], i32 [[TMP5]])
; IF-EVL-OUTLOOP-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP5]], [[EVL_BASED_IV]]
; IF-EVL-OUTLOOP-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP5]]
; IF-EVL-OUTLOOP-NEXT: [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; IF-EVL-OUTLOOP-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; IF-EVL-OUTLOOP: middle.block:
-; IF-EVL-OUTLOOP-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP10]])
+; IF-EVL-OUTLOOP-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP4]])
; IF-EVL-OUTLOOP-NEXT: br label [[FOR_BODY:%.*]]
; IF-EVL-OUTLOOP: for.cond.cleanup.loopexit:
; IF-EVL-OUTLOOP-NEXT: br label [[FOR_COND_CLEANUP]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
index 5ef175eaca194..a204725907df5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
@@ -34,20 +34,20 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV32: vector.body:
; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV32-NEXT: [[AVL:%.*]] = phi i64 [ 625, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; RV32-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; RV32-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; RV32-NEXT: [[TMP12:%.*]] = trunc i64 [[INDEX]] to i32
; RV32-NEXT: [[TMP6:%.*]] = shl nuw i32 [[TMP12]], 6
; RV32-NEXT: [[TMP13:%.*]] = getelementptr nuw i8, ptr [[TRIGGER]], i32 [[TMP6]]
-; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i32(ptr align 4 [[TMP13]], i32 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
-; RV32-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
+; RV32-NEXT: [[TMP15:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i32(ptr align 4 [[TMP13]], i32 64, <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
+; RV32-NEXT: [[TMP9:%.*]] = icmp slt <vscale x 4 x i32> [[TMP15]], splat (i32 100)
; RV32-NEXT: [[TMP20:%.*]] = shl i32 [[TMP12]], 8
; RV32-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[B]], i32 [[TMP20]]
-; RV32-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i32(ptr align 8 [[TMP25]], i32 256, <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
-; RV32-NEXT: [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
-; RV32-NEXT: [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
+; RV32-NEXT: [[TMP14:%.*]] = call <vscale x 4 x double> @llvm.experimental.vp.strided.load.nxv4f64.p0.i32(ptr align 8 [[TMP25]], i32 256, <vscale x 4 x i1> [[TMP9]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
+; RV32-NEXT: [[TMP16:%.*]] = sitofp <vscale x 4 x i32> [[TMP15]] to <vscale x 4 x double>
+; RV32-NEXT: [[TMP17:%.*]] = fadd <vscale x 4 x double> [[TMP14]], [[TMP16]]
; RV32-NEXT: [[TMP11:%.*]] = shl i32 [[TMP12]], 7
-; RV32-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[A]], i32 [[TMP11]]
-; RV32-NEXT: call void @llvm.experimental.vp.strided.store.nxv2f64.p0.i32(<vscale x 2 x double> [[TMP18]], ptr align 8 [[TMP15]], i32 128, <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
+; RV32-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[A]], i32 [[TMP11]]
+; RV32-NEXT: call void @llvm.experimental.vp.strided.store.nxv4f64.p0.i32(<vscale x 4 x double> [[TMP17]], ptr align 8 [[TMP18]], i32 128, <vscale x 4 x i1> [[TMP9]], i32 [[TMP10]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
; RV32-NEXT: [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
@@ -99,19 +99,19 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; RV64: vector.body:
; RV64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; RV64-NEXT: [[AVL:%.*]] = phi i64 [ 625, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; RV64-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; RV64-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; RV64-NEXT: [[TMP5:%.*]] = shl nuw i64 [[INDEX]], 6
; RV64-NEXT: [[TMP6:%.*]] = getelementptr nuw i8, ptr [[TRIGGER]], i64 [[TMP5]]
-; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP6]], i64 64, <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
-; RV64-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
+; RV64-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP6]], i64 64, <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
+; RV64-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 4 x i32> [[TMP7]], splat (i32 100)
; RV64-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 8
; RV64-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP9]]
-; RV64-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i64(ptr align 8 [[TMP11]], i64 256, <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
-; RV64-NEXT: [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
-; RV64-NEXT: [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
+; RV64-NEXT: [[TMP15:%.*]] = call <vscale x 4 x double> @llvm.experimental.vp.strided.load.nxv4f64.p0.i64(ptr align 8 [[TMP11]], i64 256, <vscale x 4 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
+; RV64-NEXT: [[TMP16:%.*]] = sitofp <vscale x 4 x i32> [[TMP7]] to <vscale x 4 x double>
+; RV64-NEXT: [[TMP17:%.*]] = fadd <vscale x 4 x double> [[TMP15]], [[TMP16]]
; RV64-NEXT: [[TMP12:%.*]] = shl i64 [[INDEX]], 7
; RV64-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP12]]
-; RV64-NEXT: call void @llvm.experimental.vp.strided.store.nxv2f64.p0.i64(<vscale x 2 x double> [[TMP18]], ptr align 8 [[TMP13]], i64 128, <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
+; RV64-NEXT: call void @llvm.experimental.vp.strided.store.nxv4f64.p0.i64(<vscale x 4 x double> [[TMP17]], ptr align 8 [[TMP13]], i64 128, <vscale x 4 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
; RV64-NEXT: [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; RV64-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
index 769add4e2a1b3..2aa46ca63fad5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
@@ -15,25 +15,25 @@ define i32 @vdot4a(ptr %a, ptr %b) #0 {
; V-NEXT: br label [[VECTOR_BODY:%.*]]
; V: vector.body:
; V-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; V-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; V-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
; V-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; V-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; V-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; V-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; V-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; V-NEXT: [[TMP8:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; V-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; V-NEXT: [[TMP2:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
; V-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; V-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; V-NEXT: [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
-; V-NEXT: [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
-; V-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
-; V-NEXT: [[TMP13]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; V-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; V-NEXT: [[TMP4:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; V-NEXT: [[TMP5:%.*]] = mul <vscale x 16 x i32> [[TMP4]], [[TMP2]]
+; V-NEXT: [[TMP8:%.*]] = add <vscale x 16 x i32> [[TMP5]], [[VEC_PHI]]
+; V-NEXT: [[TMP7]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> [[VEC_PHI]], i32 [[TMP0]])
; V-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
; V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
; V-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; V-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; V-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; V: middle.block:
-; V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])
+; V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP7]])
; V-NEXT: br label [[FOR_EXIT:%.*]]
; V: for.exit:
; V-NEXT: ret i32 [[TMP15]]
@@ -46,25 +46,25 @@ define i32 @vdot4a(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
; ZVDOT4A8I: vector.body:
; ZVDOT4A8I-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; ZVDOT4A8I-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; ZVDOT4A8I-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; ZVDOT4A8I-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
; ZVDOT4A8I-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; ZVDOT4A8I-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; ZVDOT4A8I-NEXT: [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
-; ZVDOT4A8I-NEXT: [[TMP8:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
-; ZVDOT4A8I-NEXT: [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
-; ZVDOT4A8I-NEXT: [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
-; ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP10]])
+; ZVDOT4A8I-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; ZVDOT4A8I-NEXT: [[TMP4:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
+; ZVDOT4A8I-NEXT: [[TMP5:%.*]] = mul <vscale x 16 x i32> [[TMP3]], [[TMP4]]
+; ZVDOT4A8I-NEXT: [[TMP8:%.*]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP5]], <vscale x 16 x i32> zeroinitializer, i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
; ZVDOT4A8I-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
; ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
; ZVDOT4A8I-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; ZVDOT4A8I-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; ZVDOT4A8I-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; ZVDOT4A8I: middle.block:
-; ZVDOT4A8I-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
+; ZVDOT4A8I-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; ZVDOT4A8I-NEXT: br label [[FOR_EXIT:%.*]]
; ZVDOT4A8I: for.exit:
; ZVDOT4A8I-NEXT: ret i32 [[TMP14]]
@@ -77,20 +77,20 @@ define i32 @vdot4a(ptr %a, ptr %b) #0 {
; FIXED-V-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED-V: vector.body:
; FIXED-V-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-V-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; FIXED-V-NEXT: [[VEC_PHI:%.*]] = phi <32 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
; FIXED-V-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; FIXED-V-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
-; FIXED-V-NEXT: [[TMP4:%.*]] = sext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
+; FIXED-V-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP0]], align 1
+; FIXED-V-NEXT: [[TMP1:%.*]] = sext <32 x i8> [[WIDE_LOAD]] to <32 x i32>
; FIXED-V-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; FIXED-V-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
-; FIXED-V-NEXT: [[TMP9:%.*]] = sext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
-; FIXED-V-NEXT: [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP4]]
-; FIXED-V-NEXT: [[TMP13]] = add <8 x i32> [[TMP11]], [[VEC_PHI1]]
-; FIXED-V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-V-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP5]], align 1
+; FIXED-V-NEXT: [[TMP3:%.*]] = sext <32 x i8> [[WIDE_LOAD1]] to <32 x i32>
+; FIXED-V-NEXT: [[TMP4:%.*]] = mul <32 x i32> [[TMP3]], [[TMP1]]
+; FIXED-V-NEXT: [[TMP6]] = add <32 x i32> [[TMP4]], [[VEC_PHI]]
+; FIXED-V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; FIXED-V-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXED-V-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; FIXED-V: middle.block:
-; FIXED-V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP13]])
+; FIXED-V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> [[TMP6]])
; FIXED-V-NEXT: br label [[FOR_EXIT:%.*]]
; FIXED-V: for.exit:
; FIXED-V-NEXT: ret i32 [[TMP15]]
@@ -103,20 +103,20 @@ define i32 @vdot4a(ptr %a, ptr %b) #0 {
; FIXED-ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED-ZVDOT4A8I: vector.body:
; FIXED-ZVDOT4A8I-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-ZVDOT4A8I-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
+; FIXED-ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; FIXED-ZVDOT4A8I-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
+; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP0]], align 1
; FIXED-ZVDOT4A8I-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
-; FIXED-ZVDOT4A8I-NEXT: [[TMP9:%.*]] = sext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
-; FIXED-ZVDOT4A8I-NEXT: [[TMP12:%.*]] = sext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
-; FIXED-ZVDOT4A8I-NEXT: [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP12]]
-; FIXED-ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE5]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP11]])
-; FIXED-ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP5]], align 1
+; FIXED-ZVDOT4A8I-NEXT: [[TMP2:%.*]] = sext <32 x i8> [[WIDE_LOAD1]] to <32 x i32>
+; FIXED-ZVDOT4A8I-NEXT: [[TMP3:%.*]] = sext <32 x i8> [[WIDE_LOAD]] to <32 x i32>
+; FIXED-ZVDOT4A8I-NEXT: [[TMP4:%.*]] = mul <32 x i32> [[TMP2]], [[TMP3]]
+; FIXED-ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v32i32(<8 x i32> [[VEC_PHI]], <32 x i32> [[TMP4]])
+; FIXED-ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; FIXED-ZVDOT4A8I-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXED-ZVDOT4A8I-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; FIXED-ZVDOT4A8I: middle.block:
-; FIXED-ZVDOT4A8I-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE5]])
+; FIXED-ZVDOT4A8I-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[PARTIAL_REDUCE]])
; FIXED-ZVDOT4A8I-NEXT: br label [[FOR_EXIT:%.*]]
; FIXED-ZVDOT4A8I: for.exit:
; FIXED-ZVDOT4A8I-NEXT: ret i32 [[TMP13]]
@@ -153,25 +153,25 @@ define i32 @vdot4au(ptr %a, ptr %b) #0 {
; V-NEXT: br label [[VECTOR_BODY:%.*]]
; V: vector.body:
; V-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; V-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; V-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
; V-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; V-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; V-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; V-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; V-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; V-NEXT: [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; V-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; V-NEXT: [[TMP2:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
; V-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; V-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; V-NEXT: [[TMP11:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
-; V-NEXT: [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
-; V-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
-; V-NEXT: [[TMP13]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; V-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; V-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; V-NEXT: [[TMP5:%.*]] = mul <vscale x 16 x i32> [[TMP4]], [[TMP2]]
+; V-NEXT: [[TMP8:%.*]] = add <vscale x 16 x i32> [[TMP5]], [[VEC_PHI]]
+; V-NEXT: [[TMP7]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> [[VEC_PHI]], i32 [[TMP0]])
; V-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
; V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
; V-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; V-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; V-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; V: middle.block:
-; V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])
+; V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP7]])
; V-NEXT: br label [[FOR_EXIT:%.*]]
; V: for.exit:
; V-NEXT: ret i32 [[TMP15]]
@@ -184,25 +184,25 @@ define i32 @vdot4au(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
; ZVDOT4A8I: vector.body:
; ZVDOT4A8I-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; ZVDOT4A8I-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; ZVDOT4A8I-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; ZVDOT4A8I-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
; ZVDOT4A8I-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; ZVDOT4A8I-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; ZVDOT4A8I-NEXT: [[TMP11:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
-; ZVDOT4A8I-NEXT: [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
-; ZVDOT4A8I-NEXT: [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
-; ZVDOT4A8I-NEXT: [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
-; ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP10]])
+; ZVDOT4A8I-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; ZVDOT4A8I-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
+; ZVDOT4A8I-NEXT: [[TMP5:%.*]] = mul <vscale x 16 x i32> [[TMP3]], [[TMP4]]
+; ZVDOT4A8I-NEXT: [[TMP8:%.*]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP5]], <vscale x 16 x i32> zeroinitializer, i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
; ZVDOT4A8I-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
; ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
; ZVDOT4A8I-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; ZVDOT4A8I-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; ZVDOT4A8I-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; ZVDOT4A8I: middle.block:
-; ZVDOT4A8I-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
+; ZVDOT4A8I-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; ZVDOT4A8I-NEXT: br label [[FOR_EXIT:%.*]]
; ZVDOT4A8I: for.exit:
; ZVDOT4A8I-NEXT: ret i32 [[TMP14]]
@@ -215,20 +215,20 @@ define i32 @vdot4au(ptr %a, ptr %b) #0 {
; FIXED-V-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED-V: vector.body:
; FIXED-V-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-V-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; FIXED-V-NEXT: [[VEC_PHI:%.*]] = phi <32 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
; FIXED-V-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; FIXED-V-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
-; FIXED-V-NEXT: [[TMP4:%.*]] = zext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
+; FIXED-V-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP0]], align 1
+; FIXED-V-NEXT: [[TMP1:%.*]] = zext <32 x i8> [[WIDE_LOAD]] to <32 x i32>
; FIXED-V-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; FIXED-V-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
-; FIXED-V-NEXT: [[TMP9:%.*]] = zext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
-; FIXED-V-NEXT: [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP4]]
-; FIXED-V-NEXT: [[TMP13]] = add <8 x i32> [[TMP11]], [[VEC_PHI1]]
-; FIXED-V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-V-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP5]], align 1
+; FIXED-V-NEXT: [[TMP3:%.*]] = zext <32 x i8> [[WIDE_LOAD1]] to <32 x i32>
+; FIXED-V-NEXT: [[TMP4:%.*]] = mul <32 x i32> [[TMP3]], [[TMP1]]
+; FIXED-V-NEXT: [[TMP6]] = add <32 x i32> [[TMP4]], [[VEC_PHI]]
+; FIXED-V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; FIXED-V-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXED-V-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; FIXED-V: middle.block:
-; FIXED-V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP13]])
+; FIXED-V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> [[TMP6]])
; FIXED-V-NEXT: br label [[FOR_EXIT:%.*]]
; FIXED-V: for.exit:
; FIXED-V-NEXT: ret i32 [[TMP15]]
@@ -241,20 +241,20 @@ define i32 @vdot4au(ptr %a, ptr %b) #0 {
; FIXED-ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED-ZVDOT4A8I: vector.body:
; FIXED-ZVDOT4A8I-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-ZVDOT4A8I-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
+; FIXED-ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; FIXED-ZVDOT4A8I-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
+; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP0]], align 1
; FIXED-ZVDOT4A8I-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
-; FIXED-ZVDOT4A8I-NEXT: [[TMP9:%.*]] = zext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
-; FIXED-ZVDOT4A8I-NEXT: [[TMP12:%.*]] = zext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
-; FIXED-ZVDOT4A8I-NEXT: [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP12]]
-; FIXED-ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE5]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP11]])
-; FIXED-ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP5]], align 1
+; FIXED-ZVDOT4A8I-NEXT: [[TMP2:%.*]] = zext <32 x i8> [[WIDE_LOAD1]] to <32 x i32>
+; FIXED-ZVDOT4A8I-NEXT: [[TMP3:%.*]] = zext <32 x i8> [[WIDE_LOAD]] to <32 x i32>
+; FIXED-ZVDOT4A8I-NEXT: [[TMP4:%.*]] = mul <32 x i32> [[TMP2]], [[TMP3]]
+; FIXED-ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v32i32(<8 x i32> [[VEC_PHI]], <32 x i32> [[TMP4]])
+; FIXED-ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; FIXED-ZVDOT4A8I-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXED-ZVDOT4A8I-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; FIXED-ZVDOT4A8I: middle.block:
-; FIXED-ZVDOT4A8I-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE5]])
+; FIXED-ZVDOT4A8I-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[PARTIAL_REDUCE]])
; FIXED-ZVDOT4A8I-NEXT: br label [[FOR_EXIT:%.*]]
; FIXED-ZVDOT4A8I: for.exit:
; FIXED-ZVDOT4A8I-NEXT: ret i32 [[TMP13]]
@@ -291,25 +291,25 @@ define i32 @vdot4asu(ptr %a, ptr %b) #0 {
; V-NEXT: br label [[VECTOR_BODY:%.*]]
; V: vector.body:
; V-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; V-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; V-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
; V-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; V-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; V-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; V-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; V-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; V-NEXT: [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; V-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; V-NEXT: [[TMP2:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
; V-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; V-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; V-NEXT: [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
-; V-NEXT: [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
-; V-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
-; V-NEXT: [[TMP13]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; V-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; V-NEXT: [[TMP4:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; V-NEXT: [[TMP5:%.*]] = mul <vscale x 16 x i32> [[TMP4]], [[TMP2]]
+; V-NEXT: [[TMP8:%.*]] = add <vscale x 16 x i32> [[TMP5]], [[VEC_PHI]]
+; V-NEXT: [[TMP7]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> [[VEC_PHI]], i32 [[TMP0]])
; V-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
; V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
; V-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; V-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; V-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; V: middle.block:
-; V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])
+; V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP7]])
; V-NEXT: br label [[FOR_EXIT:%.*]]
; V: for.exit:
; V-NEXT: ret i32 [[TMP15]]
@@ -322,25 +322,25 @@ define i32 @vdot4asu(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
; ZVDOT4A8I: vector.body:
; ZVDOT4A8I-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; ZVDOT4A8I-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; ZVDOT4A8I-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; ZVDOT4A8I-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
; ZVDOT4A8I-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; ZVDOT4A8I-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; ZVDOT4A8I-NEXT: [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
-; ZVDOT4A8I-NEXT: [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
-; ZVDOT4A8I-NEXT: [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
-; ZVDOT4A8I-NEXT: [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
-; ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP10]])
+; ZVDOT4A8I-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; ZVDOT4A8I-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
+; ZVDOT4A8I-NEXT: [[TMP5:%.*]] = mul <vscale x 16 x i32> [[TMP3]], [[TMP4]]
+; ZVDOT4A8I-NEXT: [[TMP8:%.*]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP5]], <vscale x 16 x i32> zeroinitializer, i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
; ZVDOT4A8I-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
; ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
; ZVDOT4A8I-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; ZVDOT4A8I-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; ZVDOT4A8I-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; ZVDOT4A8I: middle.block:
-; ZVDOT4A8I-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
+; ZVDOT4A8I-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; ZVDOT4A8I-NEXT: br label [[FOR_EXIT:%.*]]
; ZVDOT4A8I: for.exit:
; ZVDOT4A8I-NEXT: ret i32 [[TMP14]]
@@ -353,20 +353,20 @@ define i32 @vdot4asu(ptr %a, ptr %b) #0 {
; FIXED-V-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED-V: vector.body:
; FIXED-V-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-V-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; FIXED-V-NEXT: [[VEC_PHI:%.*]] = phi <32 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
; FIXED-V-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; FIXED-V-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
-; FIXED-V-NEXT: [[TMP4:%.*]] = zext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
+; FIXED-V-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP0]], align 1
+; FIXED-V-NEXT: [[TMP1:%.*]] = zext <32 x i8> [[WIDE_LOAD]] to <32 x i32>
; FIXED-V-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; FIXED-V-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
-; FIXED-V-NEXT: [[TMP9:%.*]] = sext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
-; FIXED-V-NEXT: [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP4]]
-; FIXED-V-NEXT: [[TMP13]] = add <8 x i32> [[TMP11]], [[VEC_PHI1]]
-; FIXED-V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-V-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP5]], align 1
+; FIXED-V-NEXT: [[TMP3:%.*]] = sext <32 x i8> [[WIDE_LOAD1]] to <32 x i32>
+; FIXED-V-NEXT: [[TMP4:%.*]] = mul <32 x i32> [[TMP3]], [[TMP1]]
+; FIXED-V-NEXT: [[TMP6]] = add <32 x i32> [[TMP4]], [[VEC_PHI]]
+; FIXED-V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; FIXED-V-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXED-V-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; FIXED-V: middle.block:
-; FIXED-V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP13]])
+; FIXED-V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> [[TMP6]])
; FIXED-V-NEXT: br label [[FOR_EXIT:%.*]]
; FIXED-V: for.exit:
; FIXED-V-NEXT: ret i32 [[TMP15]]
@@ -379,20 +379,20 @@ define i32 @vdot4asu(ptr %a, ptr %b) #0 {
; FIXED-ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED-ZVDOT4A8I: vector.body:
; FIXED-ZVDOT4A8I-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-ZVDOT4A8I-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
+; FIXED-ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; FIXED-ZVDOT4A8I-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
+; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP0]], align 1
; FIXED-ZVDOT4A8I-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
-; FIXED-ZVDOT4A8I-NEXT: [[TMP10:%.*]] = sext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
-; FIXED-ZVDOT4A8I-NEXT: [[TMP8:%.*]] = zext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
-; FIXED-ZVDOT4A8I-NEXT: [[TMP14:%.*]] = mul <8 x i32> [[TMP10]], [[TMP8]]
-; FIXED-ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE5]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP14]])
-; FIXED-ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP5]], align 1
+; FIXED-ZVDOT4A8I-NEXT: [[TMP2:%.*]] = sext <32 x i8> [[WIDE_LOAD1]] to <32 x i32>
+; FIXED-ZVDOT4A8I-NEXT: [[TMP3:%.*]] = zext <32 x i8> [[WIDE_LOAD]] to <32 x i32>
+; FIXED-ZVDOT4A8I-NEXT: [[TMP4:%.*]] = mul <32 x i32> [[TMP2]], [[TMP3]]
+; FIXED-ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v32i32(<8 x i32> [[VEC_PHI]], <32 x i32> [[TMP4]])
+; FIXED-ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; FIXED-ZVDOT4A8I-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXED-ZVDOT4A8I-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; FIXED-ZVDOT4A8I: middle.block:
-; FIXED-ZVDOT4A8I-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE5]])
+; FIXED-ZVDOT4A8I-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[PARTIAL_REDUCE]])
; FIXED-ZVDOT4A8I-NEXT: br label [[FOR_EXIT:%.*]]
; FIXED-ZVDOT4A8I: for.exit:
; FIXED-ZVDOT4A8I-NEXT: ret i32 [[TMP13]]
@@ -428,25 +428,25 @@ define i32 @vdot4asu2(ptr %a, ptr %b) #0 {
; V-NEXT: br label [[VECTOR_BODY:%.*]]
; V: vector.body:
; V-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; V-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; V-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
; V-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; V-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; V-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; V-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; V-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; V-NEXT: [[TMP8:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; V-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; V-NEXT: [[TMP2:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
; V-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; V-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; V-NEXT: [[TMP11:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
-; V-NEXT: [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
-; V-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
-; V-NEXT: [[TMP13]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; V-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; V-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; V-NEXT: [[TMP5:%.*]] = mul <vscale x 16 x i32> [[TMP4]], [[TMP2]]
+; V-NEXT: [[TMP8:%.*]] = add <vscale x 16 x i32> [[TMP5]], [[VEC_PHI]]
+; V-NEXT: [[TMP7]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> [[VEC_PHI]], i32 [[TMP0]])
; V-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
; V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
; V-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; V-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; V-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; V: middle.block:
-; V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])
+; V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP7]])
; V-NEXT: br label [[FOR_EXIT:%.*]]
; V: for.exit:
; V-NEXT: ret i32 [[TMP15]]
@@ -459,25 +459,25 @@ define i32 @vdot4asu2(ptr %a, ptr %b) #0 {
; ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
; ZVDOT4A8I: vector.body:
; ZVDOT4A8I-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; ZVDOT4A8I-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; ZVDOT4A8I-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; ZVDOT4A8I-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; ZVDOT4A8I-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
; ZVDOT4A8I-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; ZVDOT4A8I-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; ZVDOT4A8I-NEXT: [[TMP11:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
-; ZVDOT4A8I-NEXT: [[TMP8:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
-; ZVDOT4A8I-NEXT: [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
-; ZVDOT4A8I-NEXT: [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
-; ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP10]])
+; ZVDOT4A8I-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[TMP3:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; ZVDOT4A8I-NEXT: [[TMP4:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
+; ZVDOT4A8I-NEXT: [[TMP5:%.*]] = mul <vscale x 16 x i32> [[TMP3]], [[TMP4]]
+; ZVDOT4A8I-NEXT: [[TMP8:%.*]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP5]], <vscale x 16 x i32> zeroinitializer, i32 [[TMP0]])
+; ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
; ZVDOT4A8I-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
; ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
; ZVDOT4A8I-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; ZVDOT4A8I-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; ZVDOT4A8I-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; ZVDOT4A8I: middle.block:
-; ZVDOT4A8I-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
+; ZVDOT4A8I-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; ZVDOT4A8I-NEXT: br label [[FOR_EXIT:%.*]]
; ZVDOT4A8I: for.exit:
; ZVDOT4A8I-NEXT: ret i32 [[TMP14]]
@@ -490,20 +490,20 @@ define i32 @vdot4asu2(ptr %a, ptr %b) #0 {
; FIXED-V-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED-V: vector.body:
; FIXED-V-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-V-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; FIXED-V-NEXT: [[VEC_PHI:%.*]] = phi <32 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
; FIXED-V-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; FIXED-V-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
-; FIXED-V-NEXT: [[TMP4:%.*]] = sext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
+; FIXED-V-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP0]], align 1
+; FIXED-V-NEXT: [[TMP1:%.*]] = sext <32 x i8> [[WIDE_LOAD]] to <32 x i32>
; FIXED-V-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; FIXED-V-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
-; FIXED-V-NEXT: [[TMP9:%.*]] = zext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
-; FIXED-V-NEXT: [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP4]]
-; FIXED-V-NEXT: [[TMP13]] = add <8 x i32> [[TMP11]], [[VEC_PHI1]]
-; FIXED-V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-V-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP5]], align 1
+; FIXED-V-NEXT: [[TMP3:%.*]] = zext <32 x i8> [[WIDE_LOAD1]] to <32 x i32>
+; FIXED-V-NEXT: [[TMP4:%.*]] = mul <32 x i32> [[TMP3]], [[TMP1]]
+; FIXED-V-NEXT: [[TMP6]] = add <32 x i32> [[TMP4]], [[VEC_PHI]]
+; FIXED-V-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; FIXED-V-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXED-V-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; FIXED-V: middle.block:
-; FIXED-V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP13]])
+; FIXED-V-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> [[TMP6]])
; FIXED-V-NEXT: br label [[FOR_EXIT:%.*]]
; FIXED-V: for.exit:
; FIXED-V-NEXT: ret i32 [[TMP15]]
@@ -516,20 +516,20 @@ define i32 @vdot4asu2(ptr %a, ptr %b) #0 {
; FIXED-ZVDOT4A8I-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED-ZVDOT4A8I: vector.body:
; FIXED-ZVDOT4A8I-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-ZVDOT4A8I-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
+; FIXED-ZVDOT4A8I-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; FIXED-ZVDOT4A8I-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
+; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP0]], align 1
; FIXED-ZVDOT4A8I-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
-; FIXED-ZVDOT4A8I-NEXT: [[TMP10:%.*]] = zext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
-; FIXED-ZVDOT4A8I-NEXT: [[TMP8:%.*]] = sext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
-; FIXED-ZVDOT4A8I-NEXT: [[TMP14:%.*]] = mul <8 x i32> [[TMP10]], [[TMP8]]
-; FIXED-ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE5]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP14]])
-; FIXED-ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-ZVDOT4A8I-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP5]], align 1
+; FIXED-ZVDOT4A8I-NEXT: [[TMP2:%.*]] = zext <32 x i8> [[WIDE_LOAD1]] to <32 x i32>
+; FIXED-ZVDOT4A8I-NEXT: [[TMP3:%.*]] = sext <32 x i8> [[WIDE_LOAD]] to <32 x i32>
+; FIXED-ZVDOT4A8I-NEXT: [[TMP4:%.*]] = mul <32 x i32> [[TMP2]], [[TMP3]]
+; FIXED-ZVDOT4A8I-NEXT: [[PARTIAL_REDUCE]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v32i32(<8 x i32> [[VEC_PHI]], <32 x i32> [[TMP4]])
+; FIXED-ZVDOT4A8I-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; FIXED-ZVDOT4A8I-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXED-ZVDOT4A8I-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; FIXED-ZVDOT4A8I: middle.block:
-; FIXED-ZVDOT4A8I-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE5]])
+; FIXED-ZVDOT4A8I-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[PARTIAL_REDUCE]])
; FIXED-ZVDOT4A8I-NEXT: br label [[FOR_EXIT:%.*]]
; FIXED-ZVDOT4A8I: for.exit:
; FIXED-ZVDOT4A8I-NEXT: ret i32 [[TMP13]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
index 2b3cf2cd7da5c..8c94187b0c786 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
@@ -11,25 +11,25 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PARTIAL_REDUCE4:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP4]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 16 x i8> [[VP_OP_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
-; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]]
-; CHECK-NEXT: [[TMP6]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]], i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP11:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 16 x i32> [[VEC_PHI]], [[TMP11]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP2]], <vscale x 16 x i32> [[TMP5]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP6]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[PREDPHI]], <vscale x 16 x i32> [[VEC_PHI]], i32 [[TMP0]])
; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP6]])
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: ret i32 [[TMP9]]
@@ -37,38 +37,28 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-FIXED-LABEL: define i32 @pred_reduction(
; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
-; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 32
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 31
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <32 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
-; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD]], zeroinitializer
; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[TMP10:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP7:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP8:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP10]]
-; CHECK-FIXED-NEXT: [[TMP12:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP7]]
-; CHECK-FIXED-NEXT: [[PREDPHI]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP8]], <8 x i32> [[VEC_PHI]]
-; CHECK-FIXED-NEXT: [[PREDPHI4]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP12]], <8 x i32> [[VEC_PHI1]]
-; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP6]], <32 x i1> [[TMP2]], <32 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP8:%.*]] = zext <32 x i8> [[WIDE_MASKED_LOAD]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = add <32 x i32> [[VEC_PHI]], [[TMP8]]
+; CHECK-FIXED-NEXT: [[PREDPHI]] = select <32 x i1> [[TMP2]], <32 x i32> [[TMP5]], <32 x i32> [[VEC_PHI]]
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-FIXED: [[MIDDLE_BLOCK]]:
-; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[PREDPHI4]], [[PREDPHI]]
-; CHECK-FIXED-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> [[PREDPHI]])
; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-FIXED: [[SCALAR_PH]]:
@@ -110,25 +100,25 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PARTIAL_REDUCE4:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP4]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 16 x i8> [[VP_OP_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
-; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]]
-; CHECK-NEXT: [[TMP6]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]], i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP11:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 16 x i32> [[VEC_PHI]], [[TMP11]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP2]], <vscale x 16 x i32> [[TMP5]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP6]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[PREDPHI]], <vscale x 16 x i32> [[VEC_PHI]], i32 [[TMP0]])
; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP6]])
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: ret i32 [[TMP9]]
@@ -136,38 +126,28 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-FIXED-LABEL: define i32 @pred_reduction_sext(
; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
-; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 32
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 31
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <32 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
-; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD]], zeroinitializer
; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[TMP10:%.*]] = sext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP7:%.*]] = sext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP8:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP10]]
-; CHECK-FIXED-NEXT: [[TMP12:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP7]]
-; CHECK-FIXED-NEXT: [[PREDPHI]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP8]], <8 x i32> [[VEC_PHI]]
-; CHECK-FIXED-NEXT: [[PREDPHI4]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP12]], <8 x i32> [[VEC_PHI1]]
-; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP6]], <32 x i1> [[TMP2]], <32 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP8:%.*]] = sext <32 x i8> [[WIDE_MASKED_LOAD]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = add <32 x i32> [[VEC_PHI]], [[TMP8]]
+; CHECK-FIXED-NEXT: [[PREDPHI]] = select <32 x i1> [[TMP2]], <32 x i32> [[TMP5]], <32 x i32> [[VEC_PHI]]
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-FIXED: [[MIDDLE_BLOCK]]:
-; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[PREDPHI4]], [[PREDPHI]]
-; CHECK-FIXED-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> [[PREDPHI]])
; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-FIXED: [[SCALAR_PH]]:
@@ -209,28 +189,28 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP4]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 16 x i8> [[VP_OP_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP2]], i32 [[TMP0]])
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP10]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
-; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP7:%.*]] = mul nuw nsw <vscale x 4 x i32> [[TMP5]], [[TMP6]]
-; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
-; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP12]])
+; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD2]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> [[TMP2]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP12]])
; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP0]] to i64
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP9]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: ret i32 [[TMP11]]
@@ -238,46 +218,46 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-FIXED-LABEL: define i32 @pred_reduction_dotprod(
; CHECK-FIXED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
-; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 64
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
-; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 32
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <32 x i8>, ptr [[TMP2]], align 1
+; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD2]], zeroinitializer
; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[TMP6]], i64 32
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP6]], <32 x i1> [[TMP3]], <32 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP17]], <32 x i1> [[TMP5]], <32 x i8> poison)
; CHECK-FIXED-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP7]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP7]], <8 x i1> [[TMP2]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP16]], <8 x i1> [[TMP3]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[TMP8:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP9:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD4]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP10:%.*]] = mul nuw nsw <8 x i32> [[TMP8]], [[TMP9]]
-; CHECK-FIXED-NEXT: [[TMP11:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP10]], <8 x i32> zeroinitializer
-; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI]], <8 x i32> [[TMP11]])
-; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP13:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD5]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP14:%.*]] = mul nuw nsw <8 x i32> [[TMP18]], [[TMP13]]
-; CHECK-FIXED-NEXT: [[TMP15:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP14]], <8 x i32> zeroinitializer
-; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE6]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP15]])
-; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 32
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP7]], <32 x i1> [[TMP3]], <32 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP8]], <32 x i1> [[TMP5]], <32 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP9:%.*]] = zext <32 x i8> [[WIDE_MASKED_LOAD]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP10:%.*]] = zext <32 x i8> [[WIDE_MASKED_LOAD4]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP11:%.*]] = mul nuw nsw <32 x i32> [[TMP9]], [[TMP10]]
+; CHECK-FIXED-NEXT: [[TMP19:%.*]] = select <32 x i1> [[TMP3]], <32 x i32> [[TMP11]], <32 x i32> zeroinitializer
+; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v32i32(<8 x i32> [[VEC_PHI]], <32 x i32> [[TMP19]])
+; CHECK-FIXED-NEXT: [[TMP13:%.*]] = zext <32 x i8> [[WIDE_MASKED_LOAD3]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP14:%.*]] = zext <32 x i8> [[WIDE_MASKED_LOAD5]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP15:%.*]] = mul nuw nsw <32 x i32> [[TMP13]], [[TMP14]]
+; CHECK-FIXED-NEXT: [[TMP16:%.*]] = select <32 x i1> [[TMP5]], <32 x i32> [[TMP15]], <32 x i32> zeroinitializer
+; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE6]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v32i32(<8 x i32> [[VEC_PHI1]], <32 x i32> [[TMP16]])
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK-FIXED: [[MIDDLE_BLOCK]]:
-; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <2 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
-; CHECK-FIXED-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
+; CHECK-FIXED-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-FIXED: [[SCALAR_PH]]:
@@ -323,28 +303,28 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP4]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 16 x i8> [[VP_OP_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP2]], i32 [[TMP0]])
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP10]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
-; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP7:%.*]] = mul nuw nsw <vscale x 4 x i32> [[TMP5]], [[TMP6]]
-; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
-; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP11]])
+; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD2]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> [[TMP2]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP0]] to i64
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP9]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; CHECK-NEXT: [[TMP22:%.*]] = sub i32 0, [[TMP21]]
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -353,46 +333,46 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-FIXED-LABEL: define i32 @pred_sub_reduction(
; CHECK-FIXED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
-; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 64
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
-; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 32
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <32 x i8>, ptr [[TMP2]], align 1
+; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD2]], zeroinitializer
; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[TMP6]], i64 32
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP6]], <32 x i1> [[TMP3]], <32 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP17]], <32 x i1> [[TMP5]], <32 x i8> poison)
; CHECK-FIXED-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP7]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP7]], <8 x i1> [[TMP2]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP16]], <8 x i1> [[TMP3]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[TMP8:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP9:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD4]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP10:%.*]] = mul nuw nsw <8 x i32> [[TMP8]], [[TMP9]]
-; CHECK-FIXED-NEXT: [[TMP11:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP10]], <8 x i32> zeroinitializer
-; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI]], <8 x i32> [[TMP11]])
-; CHECK-FIXED-NEXT: [[TMP17:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD5]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP19:%.*]] = mul nuw nsw <8 x i32> [[TMP17]], [[TMP18]]
-; CHECK-FIXED-NEXT: [[TMP15:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer
-; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE6]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP15]])
-; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 32
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP7]], <32 x i1> [[TMP3]], <32 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP8]], <32 x i1> [[TMP5]], <32 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP9:%.*]] = zext <32 x i8> [[WIDE_MASKED_LOAD]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP10:%.*]] = zext <32 x i8> [[WIDE_MASKED_LOAD4]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP11:%.*]] = mul nuw nsw <32 x i32> [[TMP9]], [[TMP10]]
+; CHECK-FIXED-NEXT: [[TMP18:%.*]] = select <32 x i1> [[TMP3]], <32 x i32> [[TMP11]], <32 x i32> zeroinitializer
+; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v32i32(<8 x i32> [[VEC_PHI]], <32 x i32> [[TMP18]])
+; CHECK-FIXED-NEXT: [[TMP19:%.*]] = zext <32 x i8> [[WIDE_MASKED_LOAD3]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP20:%.*]] = zext <32 x i8> [[WIDE_MASKED_LOAD5]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP15:%.*]] = mul nuw nsw <32 x i32> [[TMP19]], [[TMP20]]
+; CHECK-FIXED-NEXT: [[TMP16:%.*]] = select <32 x i1> [[TMP5]], <32 x i32> [[TMP15]], <32 x i32> zeroinitializer
+; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE6]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v32i32(<8 x i32> [[VEC_PHI1]], <32 x i32> [[TMP16]])
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK-FIXED: [[MIDDLE_BLOCK]]:
-; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <2 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
-; CHECK-FIXED-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
+; CHECK-FIXED-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
; CHECK-FIXED-NEXT: [[TMP14:%.*]] = sub i32 0, [[TMP13]]
; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -439,29 +419,29 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP4]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 16 x i8> [[VP_OP_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
-; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP6]]
-; CHECK-NEXT: [[PARTIAL_REDUCE8:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 16 x i32> [[VEC_PHI]], [[TMP6]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP2]], <vscale x 16 x i32> [[TMP5]], <vscale x 16 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; CHECK-NEXT: [[PARTIAL_REDUCE7:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
-; CHECK-NEXT: [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP14]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD2]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 16 x i32> [[PREDPHI]], [[TMP7]]
+; CHECK-NEXT: [[TMP9]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP13]], <vscale x 16 x i32> [[VEC_PHI]], i32 [[TMP0]])
; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP9]])
+; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP9]])
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: ret i32 [[TMP12]]
@@ -469,46 +449,32 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-FIXED-LABEL: define i32 @chained_pred_reduction(
; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
-; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 32
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 31
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <32 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
-; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD]], zeroinitializer
; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[TMP10:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP7:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP8:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP10]]
-; CHECK-FIXED-NEXT: [[TMP16:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP7]]
-; CHECK-FIXED-NEXT: [[PREDPHI:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP8]], <8 x i32> [[VEC_PHI]]
-; CHECK-FIXED-NEXT: [[PREDPHI4:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP16]], <8 x i32> [[VEC_PHI1]]
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP6]], <32 x i1> [[TMP2]], <32 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP11:%.*]] = zext <32 x i8> [[WIDE_MASKED_LOAD]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = add <32 x i32> [[VEC_PHI]], [[TMP11]]
+; CHECK-FIXED-NEXT: [[PREDPHI:%.*]] = select <32 x i1> [[TMP2]], <32 x i32> [[TMP5]], <32 x i32> [[VEC_PHI]]
; CHECK-FIXED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i8>, ptr [[TMP9]], align 1
-; CHECK-FIXED-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i8>, ptr [[TMP11]], align 1
-; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_LOAD5]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP13:%.*]] = zext <8 x i8> [[WIDE_LOAD6]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP14]] = add <8 x i32> [[PREDPHI]], [[TMP18]]
-; CHECK-FIXED-NEXT: [[TMP15]] = add <8 x i32> [[PREDPHI4]], [[TMP13]]
-; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP9]], align 1
+; CHECK-FIXED-NEXT: [[TMP7:%.*]] = zext <32 x i8> [[WIDE_LOAD1]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP8]] = add <32 x i32> [[PREDPHI]], [[TMP7]]
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-FIXED: [[MIDDLE_BLOCK]]:
-; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[TMP15]], [[TMP14]]
-; CHECK-FIXED-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> [[TMP8]])
; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-FIXED: [[SCALAR_PH]]:
@@ -554,29 +520,29 @@ define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; CHECK-NEXT: [[TMP2:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[PARTIAL_REDUCE8:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP2]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP4]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 16 x i32> [[VEC_PHI]], [[TMP2]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[VP_OP_LOAD1]], zeroinitializer
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP12]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]])
-; CHECK-NEXT: [[PARTIAL_REDUCE7:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[PARTIAL_REDUCE8]]
-; CHECK-NEXT: [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP5]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD2]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP14:%.*]] = add <vscale x 16 x i32> [[TMP3]], [[TMP7]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP14]], <vscale x 16 x i32> [[TMP3]]
+; CHECK-NEXT: [[TMP9]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[PREDPHI]], <vscale x 16 x i32> [[VEC_PHI]], i32 [[TMP0]])
; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP9]])
+; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP9]])
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: ret i32 [[TMP13]]
@@ -584,46 +550,32 @@ define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %
; CHECK-FIXED-LABEL: define i32 @reduction_before_pred(
; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
-; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 32
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 31
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <32 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
-; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; CHECK-FIXED-NEXT: [[TMP2:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP6:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP2]]
-; CHECK-FIXED-NEXT: [[TMP5:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP3]]
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = zext <32 x i8> [[WIDE_LOAD]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP3:%.*]] = add <32 x i32> [[VEC_PHI]], [[TMP2]]
; CHECK-FIXED-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1
-; CHECK-FIXED-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP10]], align 1
-; CHECK-FIXED-NEXT: [[TMP8:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD3]], zeroinitializer
-; CHECK-FIXED-NEXT: [[TMP16:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD4]], zeroinitializer
+; CHECK-FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP7]], align 1
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD1]], zeroinitializer
; CHECK-FIXED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-FIXED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP9]], i64 8
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP9]], <8 x i1> [[TMP8]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP11]], <8 x i1> [[TMP16]], <8 x i8> poison)
-; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP13:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD5]] to <8 x i32>
-; CHECK-FIXED-NEXT: [[TMP14:%.*]] = add <8 x i32> [[TMP6]], [[TMP18]]
-; CHECK-FIXED-NEXT: [[TMP15:%.*]] = add <8 x i32> [[TMP5]], [[TMP13]]
-; CHECK-FIXED-NEXT: [[PREDPHI]] = select <8 x i1> [[TMP8]], <8 x i32> [[TMP14]], <8 x i32> [[TMP6]]
-; CHECK-FIXED-NEXT: [[PREDPHI6]] = select <8 x i1> [[TMP16]], <8 x i32> [[TMP15]], <8 x i32> [[TMP5]]
-; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP9]], <32 x i1> [[TMP5]], <32 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP11:%.*]] = zext <32 x i8> [[WIDE_MASKED_LOAD]] to <32 x i32>
+; CHECK-FIXED-NEXT: [[TMP8:%.*]] = add <32 x i32> [[TMP3]], [[TMP11]]
+; CHECK-FIXED-NEXT: [[PREDPHI]] = select <32 x i1> [[TMP5]], <32 x i32> [[TMP8]], <32 x i32> [[TMP3]]
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-FIXED: [[MIDDLE_BLOCK]]:
-; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[PREDPHI6]], [[PREDPHI]]
-; CHECK-FIXED-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> [[PREDPHI]])
; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-FIXED: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce.ll
index 73b204a715a94..dc564e1164e1a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce.ll
@@ -10,25 +10,25 @@ define i32 @partial_reduce_dotprod(ptr %a, ptr %b, ptr %c, ptr %d, i64 %N) #0 {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; CHECK-NEXT: [[TMP7:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP7]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
-; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP3]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP7:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 16 x i32> [[TMP7]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP5]], <vscale x 16 x i32> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP6]])
; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP8]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 [[TMP10]]
@@ -64,34 +64,34 @@ define i32 @chained_partial_reduce_dotprod(ptr %a, ptr %b, ptr %c, ptr %d, i64 %
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP9]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
-; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP3]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP13:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 16 x i32> [[TMP13]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP5]], <vscale x 16 x i32> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP6]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[D]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD3:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; CHECK-NEXT: [[TMP13:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP10:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD3]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP11:%.*]] = mul <vscale x 4 x i32> [[TMP13]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
-; CHECK-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]], <vscale x 4 x i32> [[TMP12]])
+; CHECK-NEXT: [[VP_OP_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD2]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP10:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD3]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP11:%.*]] = mul <vscale x 16 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP11]], <vscale x 16 x i32> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP12]])
; CHECK-NEXT: [[TMP14:%.*]] = zext i32 [[TMP0]] to i64
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP14]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE4]])
+; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE4]])
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 [[TMP15]]
@@ -134,38 +134,38 @@ define i32 @chained_reduce_iv_derived_first_link(ptr %c, ptr %d, i64 %N) #0 {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.stepvector.nxv4i8()
+; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.stepvector.nxv16i8()
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i8> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i8
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i8> [[VEC_IND]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP1]])
-; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP9]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = sext <vscale x 16 x i8> [[VEC_IND]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 16 x i32> [[TMP4]], [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP5]], <vscale x 16 x i32> zeroinitializer, i32 [[TMP1]])
+; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[D]], i64 [[INDEX]]
-; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
-; CHECK-NEXT: [[TMP8:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP10:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[TMP11:%.*]] = mul <vscale x 4 x i32> [[TMP8]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP1]])
-; CHECK-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]], <vscale x 4 x i32> [[TMP12]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP10:%.*]] = sext <vscale x 16 x i8> [[VP_OP_LOAD1]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP11:%.*]] = mul <vscale x 16 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 16 x i32> @llvm.vp.merge.nxv16i32(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i32> [[TMP11]], <vscale x 16 x i32> zeroinitializer, i32 [[TMP1]])
+; CHECK-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP12]])
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP2]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP2]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 [[TMP14]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index 667612a193dd5..68de69d530a65 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -72,27 +72,27 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[START]], i64 4
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[END]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT6]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[END]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_MEMCHECK]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START]], %[[VECTOR_MEMCHECK]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP3]], %[[VECTOR_MEMCHECK]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; CHECK-NEXT: [[TMP14:%.*]] = mul <vscale x 2 x i64> [[TMP13]], splat (i64 12)
-; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP14]]
-; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT: [[TMP8:%.*]] = mul <vscale x 4 x i64> [[TMP7]], splat (i64 12)
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i64> [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[TMP26:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[INDEX]], 12
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[SCEVGEP6]], i64 [[TMP12]]
-; CHECK-NEXT: [[TMP18:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP15]], i64 12, <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
-; CHECK-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[TMP18]] to <vscale x 2 x i64>
-; CHECK-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
-; CHECK-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP21]], <vscale x 2 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP15]], i64 12, <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT: [[TMP14:%.*]] = zext <vscale x 4 x i32> [[TMP13]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP17:%.*]] = mul <vscale x 4 x i64> [[TMP14]], splat (i64 -7070675565921424023)
+; CHECK-NEXT: [[TMP16:%.*]] = add <vscale x 4 x i64> [[TMP17]], splat (i64 -4)
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i64.nxv4p0(<vscale x 4 x i64> [[TMP16]], <vscale x 4 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP26]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
; CHECK-NEXT: [[TMP27:%.*]] = mul i64 12, [[TMP26]]
@@ -100,10 +100,10 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP30:%.*]] = getelementptr nusw i8, <vscale x 2 x ptr> [[VECTOR_GEP]], i64 12
-; CHECK-NEXT: [[TMP17:%.*]] = icmp eq <vscale x 2 x ptr> [[TMP30]], [[BROADCAST_SPLAT7]]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr nusw i8, <vscale x 4 x ptr> [[VECTOR_GEP]], i64 12
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq <vscale x 4 x ptr> [[WIDE_GEP]], [[BROADCAST_SPLAT2]]
; CHECK-NEXT: [[TMP29:%.*]] = sub i64 [[TMP26]], 1
-; CHECK-NEXT: [[TMP25:%.*]] = extractelement <vscale x 2 x i1> [[TMP17]], i64 [[TMP29]]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <vscale x 4 x i1> [[TMP19]], i64 [[TMP29]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: ret i1 [[TMP25]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
index 314ec3d7a51d0..c41a1d72a8338 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
@@ -26,12 +26,12 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
; IF-EVL: [[VECTOR_BODY]]:
; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr align 4 [[TMP14]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META0:![0-9]+]]
-; IF-EVL-NEXT: [[TMP16:%.*]] = sext <vscale x 2 x i32> [[VP_OP_LOAD]] to <vscale x 2 x i64>
+; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META0:![0-9]+]]
+; IF-EVL-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i32> [[VP_OP_LOAD]] to <vscale x 4 x i64>
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP16]], ptr align 8 [[TMP17]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[TMP4]], ptr align 8 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP19]]
@@ -58,8 +58,8 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; NO-VP-NEXT: [[ENTRY:.*]]:
; NO-VP-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 1
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 20)
+; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
+; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 24)
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
@@ -78,10 +78,10 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
; NO-VP: [[VECTOR_BODY]]:
; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
-; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[GEP]], align 4, !alias.scope [[META0:![0-9]+]]
-; NO-VP-NEXT: [[TMP10:%.*]] = sext <vscale x 2 x i32> [[WIDE_LOAD]] to <vscale x 2 x i64>
+; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[GEP]], align 4, !alias.scope [[META0:![0-9]+]]
+; NO-VP-NEXT: [[TMP7:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
; NO-VP-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-VP-NEXT: store <vscale x 2 x i64> [[TMP10]], ptr [[TMP11]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; NO-VP-NEXT: store <vscale x 4 x i64> [[TMP7]], ptr [[TMP11]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; NO-VP-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; NO-VP-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -141,12 +141,12 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
; IF-EVL: [[VECTOR_BODY]]:
; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr align 4 [[TMP14]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META9:![0-9]+]]
-; IF-EVL-NEXT: [[TMP16:%.*]] = zext <vscale x 2 x i32> [[VP_OP_LOAD]] to <vscale x 2 x i64>
+; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META9:![0-9]+]]
+; IF-EVL-NEXT: [[TMP4:%.*]] = zext <vscale x 4 x i32> [[VP_OP_LOAD]] to <vscale x 4 x i64>
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP16]], ptr align 8 [[TMP17]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META12:![0-9]+]], !noalias [[META9]]
+; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[TMP4]], ptr align 8 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META12:![0-9]+]], !noalias [[META9]]
; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP19]]
@@ -173,8 +173,8 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; NO-VP-NEXT: [[ENTRY:.*]]:
; NO-VP-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 1
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 20)
+; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
+; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 24)
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
@@ -193,10 +193,10 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
; NO-VP: [[VECTOR_BODY]]:
; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
-; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[GEP]], align 4, !alias.scope [[META9:![0-9]+]]
-; NO-VP-NEXT: [[TMP10:%.*]] = zext <vscale x 2 x i32> [[WIDE_LOAD]] to <vscale x 2 x i64>
+; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[GEP]], align 4, !alias.scope [[META9:![0-9]+]]
+; NO-VP-NEXT: [[TMP7:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
; NO-VP-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-VP-NEXT: store <vscale x 2 x i64> [[TMP10]], ptr [[TMP11]], align 8, !alias.scope [[META12:![0-9]+]], !noalias [[META9]]
+; NO-VP-NEXT: store <vscale x 4 x i64> [[TMP7]], ptr [[TMP11]], align 8, !alias.scope [[META12:![0-9]+]], !noalias [[META9]]
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; NO-VP-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; NO-VP-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
@@ -256,12 +256,12 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
; IF-EVL: [[VECTOR_BODY]]:
; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP14]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META16:![0-9]+]]
-; IF-EVL-NEXT: [[TMP16:%.*]] = trunc <vscale x 2 x i64> [[VP_OP_LOAD]] to <vscale x 2 x i32>
+; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr align 8 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META16:![0-9]+]]
+; IF-EVL-NEXT: [[TMP4:%.*]] = trunc <vscale x 4 x i64> [[VP_OP_LOAD]] to <vscale x 4 x i32>
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i32.p0(<vscale x 2 x i32> [[TMP16]], ptr align 4 [[TMP17]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META19:![0-9]+]], !noalias [[META16]]
+; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP4]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META19:![0-9]+]], !noalias [[META16]]
; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP19]]
@@ -288,8 +288,8 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; NO-VP-NEXT: [[ENTRY:.*]]:
; NO-VP-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 1
-; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 20)
+; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
+; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 24)
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-VP: [[VECTOR_MEMCHECK]]:
@@ -308,10 +308,10 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
; NO-VP: [[VECTOR_BODY]]:
; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[GEP]], align 8, !alias.scope [[META16:![0-9]+]]
-; NO-VP-NEXT: [[TMP10:%.*]] = trunc <vscale x 2 x i64> [[WIDE_LOAD]] to <vscale x 2 x i32>
+; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[GEP]], align 8, !alias.scope [[META16:![0-9]+]]
+; NO-VP-NEXT: [[TMP7:%.*]] = trunc <vscale x 4 x i64> [[WIDE_LOAD]] to <vscale x 4 x i32>
; NO-VP-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; NO-VP-NEXT: store <vscale x 2 x i32> [[TMP10]], ptr [[TMP11]], align 4, !alias.scope [[META19:![0-9]+]], !noalias [[META16]]
+; NO-VP-NEXT: store <vscale x 4 x i32> [[TMP7]], ptr [[TMP11]], align 4, !alias.scope [[META19:![0-9]+]], !noalias [[META16]]
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; NO-VP-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; NO-VP-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
@@ -371,12 +371,12 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
; IF-EVL: [[VECTOR_BODY]]:
; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x float> @llvm.vp.load.nxv2f32.p0(ptr align 4 [[TMP14]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META23:![0-9]+]]
-; IF-EVL-NEXT: [[TMP16:%.*]] = fpext <vscale x 2 x float> [[VP_OP_LOAD]] to <vscale x 2 x double>
+; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META23:![0-9]+]]
+; IF-EVL-NEXT: [[TMP4:%.*]] = fpext <vscale x 4 x float> [[VP_OP_LOAD]] to <vscale x 4 x double>
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: call void @llvm.vp.store.nxv2f64.p0(<vscale x 2 x double> [[TMP16]], ptr align 8 [[TMP17]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META26:![0-9]+]], !noalias [[META23]]
+; IF-EVL-NEXT: call void @llvm.vp.store.nxv4f64.p0(<vscale x 4 x double> [[TMP4]], ptr align 8 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META26:![0-9]+]], !noalias [[META23]]
; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP19]]
@@ -403,7 +403,7 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; NO-VP-NEXT: [[ENTRY:.*]]:
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 1
+; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 2
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 16)
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -423,10 +423,10 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
; NO-VP: [[VECTOR_BODY]]:
; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x float>, ptr [[GEP]], align 4, !alias.scope [[META23:![0-9]+]]
-; NO-VP-NEXT: [[TMP10:%.*]] = fpext <vscale x 2 x float> [[WIDE_LOAD]] to <vscale x 2 x double>
+; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[GEP]], align 4, !alias.scope [[META23:![0-9]+]]
+; NO-VP-NEXT: [[TMP8:%.*]] = fpext <vscale x 4 x float> [[WIDE_LOAD]] to <vscale x 4 x double>
; NO-VP-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]]
-; NO-VP-NEXT: store <vscale x 2 x double> [[TMP10]], ptr [[TMP11]], align 8, !alias.scope [[META26:![0-9]+]], !noalias [[META23]]
+; NO-VP-NEXT: store <vscale x 4 x double> [[TMP8]], ptr [[TMP11]], align 8, !alias.scope [[META26:![0-9]+]], !noalias [[META23]]
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; NO-VP-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; NO-VP-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
@@ -486,12 +486,12 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
; IF-EVL: [[VECTOR_BODY]]:
; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x double> @llvm.vp.load.nxv2f64.p0(ptr align 8 [[TMP14]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META30:![0-9]+]]
-; IF-EVL-NEXT: [[TMP16:%.*]] = fptrunc <vscale x 2 x double> [[VP_OP_LOAD]] to <vscale x 2 x float>
+; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x double> @llvm.vp.load.nxv4f64.p0(ptr align 8 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META30:![0-9]+]]
+; IF-EVL-NEXT: [[TMP4:%.*]] = fptrunc <vscale x 4 x double> [[VP_OP_LOAD]] to <vscale x 4 x float>
; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: call void @llvm.vp.store.nxv2f32.p0(<vscale x 2 x float> [[TMP16]], ptr align 4 [[TMP17]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META33:![0-9]+]], !noalias [[META30]]
+; IF-EVL-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP4]], ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]]), !alias.scope [[META33:![0-9]+]], !noalias [[META30]]
; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64
; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP19]]
@@ -518,7 +518,7 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; NO-VP-NEXT: [[ENTRY:.*]]:
; NO-VP-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 1
+; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 2
; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 16)
; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -538,10 +538,10 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
; NO-VP: [[VECTOR_BODY]]:
; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[IV]]
-; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[GEP]], align 8, !alias.scope [[META30:![0-9]+]]
-; NO-VP-NEXT: [[TMP10:%.*]] = fptrunc <vscale x 2 x double> [[WIDE_LOAD]] to <vscale x 2 x float>
+; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x double>, ptr [[GEP]], align 8, !alias.scope [[META30:![0-9]+]]
+; NO-VP-NEXT: [[TMP8:%.*]] = fptrunc <vscale x 4 x double> [[WIDE_LOAD]] to <vscale x 4 x float>
; NO-VP-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; NO-VP-NEXT: store <vscale x 2 x float> [[TMP10]], ptr [[TMP11]], align 4, !alias.scope [[META33:![0-9]+]], !noalias [[META30]]
+; NO-VP-NEXT: store <vscale x 4 x float> [[TMP8]], ptr [[TMP11]], align 4, !alias.scope [[META33:![0-9]+]], !noalias [[META30]]
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; NO-VP-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; NO-VP-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index 1f02c3f4906b7..8cfad15c4c77c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -10,20 +10,20 @@ define void @test_pr98413_zext_removed(ptr %src, ptr noalias %dst, i64 %x) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[X]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = trunc <vscale x 8 x i64> [[BROADCAST_SPLAT]] to <vscale x 8 x i8>
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = trunc <vscale x 16 x i64> [[BROADCAST_SPLAT]] to <vscale x 16 x i8>
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 97, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 8 [[TMP8]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
-; CHECK-NEXT: [[TMP10:%.*]] = trunc <vscale x 8 x i16> [[WIDE_LOAD]] to <vscale x 8 x i8>
-; CHECK-NEXT: [[TMP11:%.*]] = and <vscale x 8 x i8> [[TMP6]], [[TMP10]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i16> @llvm.vp.load.nxv16i16.p0(ptr align 8 [[TMP8]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT: [[TMP3:%.*]] = trunc <vscale x 16 x i16> [[VP_OP_LOAD]] to <vscale x 16 x i8>
+; CHECK-NEXT: [[TMP4:%.*]] = and <vscale x 16 x i8> [[TMP0]], [[TMP3]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT: call void @llvm.vp.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP11]], ptr align 1 [[TMP12]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP4]], ptr align 1 [[TMP12]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP13]])
; CHECK-NEXT: [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[TMP7]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
@@ -60,20 +60,20 @@ define void @test_pr98413_sext_removed(ptr %src, ptr noalias %dst, i64 %x) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[X]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = trunc <vscale x 8 x i64> [[BROADCAST_SPLAT]] to <vscale x 8 x i8>
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = trunc <vscale x 16 x i64> [[BROADCAST_SPLAT]] to <vscale x 16 x i8>
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 97, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 8 [[TMP8]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
-; CHECK-NEXT: [[TMP10:%.*]] = trunc <vscale x 8 x i16> [[WIDE_LOAD]] to <vscale x 8 x i8>
-; CHECK-NEXT: [[TMP11:%.*]] = and <vscale x 8 x i8> [[TMP6]], [[TMP10]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i16> @llvm.vp.load.nxv16i16.p0(ptr align 8 [[TMP8]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT: [[TMP3:%.*]] = trunc <vscale x 16 x i16> [[VP_OP_LOAD]] to <vscale x 16 x i8>
+; CHECK-NEXT: [[TMP4:%.*]] = and <vscale x 16 x i8> [[TMP0]], [[TMP3]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT: call void @llvm.vp.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP11]], ptr align 1 [[TMP12]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP4]], ptr align 1 [[TMP12]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP13]])
; CHECK-NEXT: [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[TMP7]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
@@ -277,22 +277,22 @@ define void @test_minbws_for_trunc(i32 %n, ptr noalias %p1, ptr noalias %p2) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 256, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[INDEX]] to i64
; CHECK-NEXT: [[TMP22:%.*]] = shl i64 [[TMP13]], 4
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP22]]
-; CHECK-NEXT: [[TMP15:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr align 4 [[TMP14]], i64 16, <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
-; CHECK-NEXT: [[TMP16:%.*]] = trunc <vscale x 2 x i32> [[TMP15]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.experimental.vp.strided.load.nxv16i32.p0.i64(ptr align 4 [[TMP14]], i64 16, <vscale x 16 x i1> splat (i1 true), i32 [[TMP9]])
+; CHECK-NEXT: [[TMP7:%.*]] = trunc <vscale x 16 x i32> [[TMP6]] to <vscale x 16 x i16>
; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP13]], 3
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP8]]
-; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i16.p0.i64(<vscale x 2 x i16> [[TMP16]], ptr align 2 [[TMP10]], i64 8, <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]]), !alias.scope [[META6:![0-9]+]], !noalias [[META9:![0-9]+]]
-; CHECK-NEXT: [[TMP18:%.*]] = trunc <vscale x 2 x i32> [[TMP15]] to <vscale x 2 x i8>
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv16i16.p0.i64(<vscale x 16 x i16> [[TMP7]], ptr align 2 [[TMP10]], i64 8, <vscale x 16 x i1> splat (i1 true), i32 [[TMP9]]), !alias.scope [[META6:![0-9]+]], !noalias [[META9:![0-9]+]]
+; CHECK-NEXT: [[TMP15:%.*]] = trunc <vscale x 16 x i32> [[TMP6]] to <vscale x 16 x i8>
; CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[TMP13]], 2
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP11]]
-; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i8.p0.i64(<vscale x 2 x i8> [[TMP18]], ptr align 1 [[TMP12]], i64 4, <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]]), !alias.scope [[META12:![0-9]+]], !noalias [[META13:![0-9]+]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv16i8.p0.i64(<vscale x 16 x i8> [[TMP15]], ptr align 1 [[TMP12]], i64 4, <vscale x 16 x i1> splat (i1 true), i32 [[TMP9]]), !alias.scope [[META12:![0-9]+]], !noalias [[META13:![0-9]+]]
; CHECK-NEXT: [[TMP17:%.*]] = shl i64 [[TMP13]], 5
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP17]]
-; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i64.p0.i64(<vscale x 2 x i64> zeroinitializer, ptr align 8 [[TMP19]], i64 32, <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]]), !alias.scope [[META13]]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv16i64.p0.i64(<vscale x 16 x i64> zeroinitializer, ptr align 8 [[TMP19]], i64 32, <vscale x 16 x i1> splat (i1 true), i32 [[TMP9]]), !alias.scope [[META13]]
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP9]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP9]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
index 0d34b2b6d2d7b..c09ba3c91a1e4 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
@@ -51,7 +51,7 @@ define void @truncate_i16_to_i8_cse(ptr noalias %src, ptr noalias %dst) {
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4294967296, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -59,13 +59,13 @@ define void @truncate_i16_to_i8_cse(ptr noalias %src, ptr noalias %dst) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[SRC]], align 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = trunc <vscale x 8 x i16> [[BROADCAST_SPLAT]] to <vscale x 8 x i8>
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i16> poison, i16 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i16> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = trunc <vscale x 16 x i16> [[BROADCAST_SPLAT]] to <vscale x 16 x i8>
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 8
+; CHECK-NEXT: [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 16
; CHECK-NEXT: [[TMP9:%.*]] = sub i32 [[TMP8]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 8 x i8> [[TMP6]], i32 [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 16 x i8> [[TMP4]], i32 [[TMP9]]
; CHECK-NEXT: store i8 [[TMP10]], ptr null, align 1
; CHECK-NEXT: store i8 [[TMP10]], ptr [[DST]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
index 06d74a77bb0e3..e7bbfb7f0d7d1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
@@ -19,22 +19,22 @@ define void @type_info_cache_clobber(ptr %dstv, ptr %src, i64 %wide.trip.count)
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[DSTV]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x ptr> poison, ptr [[DSTV]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 16 x ptr> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[EVL_BASED_IV]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i8> @llvm.vp.load.nxv8i8.p0(ptr align 1 [[TMP13]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP15:%.*]] = zext <vscale x 8 x i8> [[VP_OP_LOAD]] to <vscale x 8 x i32>
-; CHECK-NEXT: [[TMP23:%.*]] = ashr <vscale x 8 x i32> [[TMP15]], zeroinitializer
-; CHECK-NEXT: [[TMP16:%.*]] = icmp ult <vscale x 8 x i32> [[TMP15]], zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = select <vscale x 8 x i1> [[TMP16]], <vscale x 8 x i32> [[TMP23]], <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP24:%.*]] = trunc <vscale x 8 x i32> [[TMP17]] to <vscale x 8 x i8>
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP24]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 splat (ptr null), <vscale x 8 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP13]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[VP_OP_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP9:%.*]] = ashr <vscale x 16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ult <vscale x 16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP9]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = trunc <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i8>
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP8]], <vscale x 16 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i16.nxv16p0(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x ptr> align 2 splat (ptr null), <vscale x 16 x i1> splat (i1 true), i32 [[TMP11]])
; CHECK-NEXT: [[TMP20:%.*]] = zext i32 [[TMP11]] to i64
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP20]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP20]]
>From 32a2e663b3e67f8a03c7a3c58a80641e297534fe Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Mon, 7 Sep 2026 16:57:13 +0800
Subject: [PATCH 2/3] [VPlan][TTI] Model register reuse at definition points
Add a conservative VPlan register-usage mode that samples result pressure at operation definitions. Targets can describe which operands may share storage with a result through the existing register-reuse query, while VPlan owns selection of the estimation mode, liveness, register classes, and duplicate operand uses.
Keep LiveIntervals as the default so existing pressure and spill-cost users retain their current behavior.
Assisted-by: TRAE CLI (GPT-5)
---
.../llvm/Analysis/TargetTransformInfo.h | 25 +-
.../llvm/Analysis/TargetTransformInfoImpl.h | 6 +
llvm/lib/Analysis/TargetTransformInfo.cpp | 6 +
.../Transforms/Vectorize/VPlanAnalysis.cpp | 142 +++++++++--
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 18 +-
.../Transforms/Vectorize/CMakeLists.txt | 1 +
.../Vectorize/VPlanAnalysisTest.cpp | 233 ++++++++++++++++++
7 files changed, 407 insertions(+), 24 deletions(-)
create mode 100644 llvm/unittests/Transforms/Vectorize/VPlanAnalysisTest.cpp
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 5e22b890ec975..93c6185a8fdeb 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -24,6 +24,7 @@
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/Uniformity.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/InterestingMemoryOperand.h"
@@ -64,7 +65,6 @@ class ProfileSummaryInfo;
class RecurrenceDescriptor;
class SCEV;
class ScalarEvolution;
-class SmallBitVector;
class StoreInst;
class SwitchInst;
class TargetLibraryInfo;
@@ -1033,6 +1033,29 @@ class TargetTransformInfo {
/// Returns the estimated number of registers required to represent \p Ty.
LLVM_ABI unsigned getRegUsageForType(Type *Ty) const;
+ /// Describes an operand when estimating register pressure at the definition
+ /// of a vectorized operation.
+ struct RegisterUsageOperandInfo {
+ /// The type of the vectorized operand.
+ Type *ValueType = nullptr;
+ /// The source type if the operand is defined by a cast, or null otherwise.
+ Type *SourceType = nullptr;
+ /// The IR opcode defining the operand, or zero if it is unknown.
+ unsigned DefOpcode = 0;
+ /// Whether the operand is uniform across all vector lanes and parts.
+ bool IsUniform = false;
+ };
+
+ /// Return the operands of the vectorized operation described by IR
+ /// \p Opcode, \p ResultType, and \p Operands whose register storage may
+ /// be reused for the result. A set bit identifies a reusable operand, an
+ /// empty mask means that the result requires separate register storage from
+ /// all operands, and std::nullopt means that the target does not model the
+ /// operation. A returned mask must contain one bit per operand.
+ LLVM_ABI std::optional<SmallBitVector>
+ getResultRegisterReuseMask(unsigned Opcode, Type *ResultType,
+ ArrayRef<RegisterUsageOperandInfo> Operands) const;
+
/// Return true if switches should be turned into lookup tables for the
/// target.
LLVM_ABI bool shouldBuildLookupTables() const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 4240eb090e952..1e0f96c1eedf0 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -469,6 +469,12 @@ class LLVM_ABI TargetTransformInfoImplBase {
virtual unsigned getRegUsageForType(Type *Ty) const { return 1; }
+ virtual std::optional<SmallBitVector> getResultRegisterReuseMask(
+ unsigned Opcode, Type *ResultType,
+ ArrayRef<TTI::RegisterUsageOperandInfo> Operands) const {
+ return std::nullopt;
+ }
+
virtual bool shouldBuildLookupTables() const { return true; }
virtual bool shouldBuildLookupTablesForConstant(Constant *C) const {
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 67dee54028c0e..ef11fc1265e6b 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -607,6 +607,12 @@ unsigned TargetTransformInfo::getRegUsageForType(Type *Ty) const {
return TTIImpl->getRegUsageForType(Ty);
}
+std::optional<SmallBitVector> TargetTransformInfo::getResultRegisterReuseMask(
+ unsigned Opcode, Type *ResultType,
+ ArrayRef<RegisterUsageOperandInfo> Operands) const {
+ return TTIImpl->getResultRegisterReuseMask(Opcode, ResultType, Operands);
+}
+
bool TargetTransformInfo::shouldBuildLookupTables() const {
return TTIImpl->shouldBuildLookupTables();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 2abfaec06dbef..a29781b89e142 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -77,6 +77,16 @@ bool VPDominatorTree::properlyDominates(const VPRecipeBase *A,
return Base::properlyDominates(ParentA, ParentB);
}
+bool VPRegisterUsage::exceedsMaxNumRegs(unsigned ClassID,
+ const TargetTransformInfo &TTI,
+ unsigned OverrideMaxNumRegs) const {
+ unsigned AvailableRegs = OverrideMaxNumRegs > 0
+ ? OverrideMaxNumRegs
+ : TTI.getNumberOfRegisters(ClassID);
+ return MaxLocalUsers.lookup(ClassID) + LoopInvariantRegs.lookup(ClassID) >
+ AvailableRegs;
+}
+
InstructionCost
VPRegisterUsage::spillCost(const TargetTransformInfo &TTI,
TargetTransformInfo::TargetCostKind CostKind,
@@ -105,7 +115,10 @@ VPRegisterUsage::spillCost(const TargetTransformInfo &TTI,
SmallVector<VPRegisterUsage, 8>
llvm::calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef<ElementCount> VFs,
- const TargetTransformInfo &TTI) {
+ const TargetTransformInfo &TTI,
+ VPRegisterUsageMode Mode) {
+ bool IncludeDefinitionPressure =
+ Mode == VPRegisterUsageMode::ConservativePeak;
DenseSet<VPRecipeBase *> EphemeralRecipes;
collectEphemeralRecipesForVPlan(Plan, EphemeralRecipes);
@@ -185,12 +198,21 @@ llvm::calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef<ElementCount> VFs,
SmallVector<VPRegisterUsage, 8> RUs(VFs.size());
SmallVector<SmallMapVector<unsigned, unsigned, 4>, 8> MaxUsages(VFs.size());
- LLVM_DEBUG(dbgs() << "LV(REG): Calculating max register usage:\n");
+ LLVM_DEBUG({
+ dbgs() << "LV(REG): Calculating ";
+ if (IncludeDefinitionPressure)
+ dbgs() << "conservative ";
+ dbgs() << "max register usage:\n";
+ });
const auto &TTICapture = TTI;
- auto GetRegUsage = [&TTICapture](Type *Ty, ElementCount VF) -> unsigned {
+ auto GetRegUsage = [&TTICapture, IncludeDefinitionPressure](
+ Type *Ty, ElementCount VF) -> unsigned {
+ // Conservative mode also accounts for scalable i1 predicates that consume
+ // target mask registers even when i1 is not a legal scalable data element.
if (Ty->isTokenTy() || !VectorType::isValidElementType(Ty) ||
(VF.isScalable() &&
+ !(IncludeDefinitionPressure && Ty->isIntegerTy(1)) &&
!TTICapture.isElementTypeLegalForScalableVector(Ty)))
return 0;
return TTICapture.getRegUsageForType(VectorType::get(Ty, VF));
@@ -235,7 +257,8 @@ llvm::calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef<ElementCount> VFs,
// there is no previous entry for ClassID.
SmallMapVector<unsigned, unsigned, 4> RegUsage;
- for (auto *VPV : OpenIntervals) {
+ auto GetRegisterUsage =
+ [&](VPValue *VPV) -> std::optional<std::pair<unsigned, unsigned>> {
// Skip artificial values or values that weren't present in the original
// loop.
// TODO: Remove skipping values that weren't present in the original
@@ -244,7 +267,7 @@ llvm::calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef<ElementCount> VFs,
if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe,
VPBranchOnMaskRecipe>(VPV) ||
match(VPV, m_ExtractLastPart(m_VPValue())))
- continue;
+ return std::nullopt;
if (VFs[J].isScalar() || VPV == CanIV ||
isa<VPReplicateRecipe, VPDerivedIVRecipe,
@@ -256,22 +279,103 @@ llvm::calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef<ElementCount> VFs,
TTI.getRegisterClassForType(false, VPV->getScalarType());
// FIXME: The target might use more than one register for the type
// even in the scalar case.
- RegUsage[ClassID] += 1;
- } else {
- // The output from scaled phis and scaled reductions actually has
- // fewer lanes than the VF.
- unsigned ScaleFactor =
- vputils::getVFScaleFactor(VPV->getDefiningRecipe());
- ElementCount VF = VFs[J];
- if (ScaleFactor > 1) {
- VF = VFs[J].divideCoefficientBy(ScaleFactor);
- LLVM_DEBUG(dbgs() << "LV(REG): Scaled down VF from " << VFs[J]
- << " to " << VF << " for " << *R << "\n";);
+ return std::make_pair(ClassID, 1);
+ }
+
+ // The output from scaled phis and scaled reductions actually has fewer
+ // lanes than the VF.
+ unsigned ScaleFactor =
+ vputils::getVFScaleFactor(VPV->getDefiningRecipe());
+ ElementCount VF = VFs[J];
+ if (ScaleFactor > 1) {
+ VF = VFs[J].divideCoefficientBy(ScaleFactor);
+ LLVM_DEBUG(dbgs() << "LV(REG): Scaled down VF from " << VFs[J]
+ << " to " << VF << " for " << *R << "\n";);
+ }
+
+ Type *ScalarTy = VPV->getScalarType();
+ unsigned ClassID = TTI.getRegisterClassForType(true, ScalarTy);
+ return std::make_pair(ClassID, GetRegUsage(ScalarTy, VF));
+ };
+
+ auto AddRegisterUsage = [&](VPValue *VPV) {
+ if (auto Usage = GetRegisterUsage(VPV))
+ RegUsage[Usage->first] += Usage->second;
+ };
+
+ for (auto *VPV : OpenIntervals)
+ AddRegisterUsage(VPV);
+
+ // Conservative mode also samples the current result unless the target
+ // can reuse an operand whose live range ends at this recipe.
+ if (IncludeDefinitionPressure && VFs[J].isVector() &&
+ R->getNumDefinedValues() == 1 &&
+ Ends.contains(R->getVPSingleValue())) {
+ VPValue *DefV = R->getVPSingleValue();
+ unsigned Opcode = vputils::getOpcode(DefV);
+ bool HasValidTypes =
+ VectorType::isValidElementType(DefV->getScalarType()) &&
+ all_of(R->operands(), [](VPValue *Op) {
+ return VectorType::isValidElementType(Op->getScalarType());
+ });
+ if (Opcode && HasValidTypes) {
+ SmallVector<TTI::RegisterUsageOperandInfo, 2> OperandInfos;
+ for (VPValue *Op : R->operands()) {
+ Type *SourceType = nullptr;
+ unsigned DefOpcode = vputils::getOpcode(Op);
+ VPRecipeBase *DefR = Op->getDefiningRecipe();
+ Type *ScalarSourceType = nullptr;
+ if (Instruction::isCast(DefOpcode) && DefR &&
+ DefR->getNumOperands() == 1)
+ ScalarSourceType = DefR->getOperand(0)->getScalarType();
+ else if (auto *I = dyn_cast_or_null<Instruction>(
+ Op->getUnderlyingValue())) {
+ DefOpcode = I->getOpcode();
+ if (auto *CastI = dyn_cast<CastInst>(I))
+ ScalarSourceType = CastI->getSrcTy();
+ }
+ if (ScalarSourceType &&
+ VectorType::isValidElementType(ScalarSourceType))
+ SourceType = VectorType::get(ScalarSourceType, VFs[J]);
+ OperandInfos.push_back(
+ {VectorType::get(Op->getScalarType(), VFs[J]), SourceType,
+ DefOpcode, vputils::isUniformAcrossVFsAndUFs(Op)});
}
- Type *ScalarTy = VPV->getScalarType();
- unsigned ClassID = TTI.getRegisterClassForType(true, ScalarTy);
- RegUsage[ClassID] += GetRegUsage(ScalarTy, VF);
+ Type *ResultType = VectorType::get(DefV->getScalarType(), VFs[J]);
+ if (auto ReusableOperands = TTI.getResultRegisterReuseMask(
+ Opcode, ResultType, OperandInfos)) {
+ assert(ReusableOperands->size() == R->getNumOperands() &&
+ "target returned invalid reusable operand mask");
+ if (ReusableOperands->size() == R->getNumOperands()) {
+ if (auto ResultUsage = GetRegisterUsage(DefV)) {
+ auto [ResultClassID, ResultRegisters] = *ResultUsage;
+ unsigned ReusableRegisters = 0;
+ SmallPtrSet<VPValue *, 2> CheckedOperands;
+ for (unsigned I = 0, E = R->getNumOperands(); I != E; ++I) {
+ VPValue *Op = R->getOperand(I);
+ if (EndPoint.lookup(Op) != Idx + 1 ||
+ !CheckedOperands.insert(Op).second)
+ continue;
+ bool AllUsesReusable =
+ all_of(seq<unsigned>(0, E), [&](unsigned J) {
+ return R->getOperand(J) != Op ||
+ ReusableOperands->test(J);
+ });
+ if (!AllUsesReusable)
+ continue;
+ auto OperandUsage = GetRegisterUsage(Op);
+ if (!OperandUsage || OperandUsage->first != ResultClassID)
+ continue;
+ ReusableRegisters =
+ std::max(ReusableRegisters, OperandUsage->second);
+ }
+ RegUsage[ResultClassID] +=
+ ResultRegisters -
+ std::min(ResultRegisters, ReusableRegisters);
+ }
+ }
+ }
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index 33cabf54145b2..885b1eded1778 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -26,6 +26,9 @@ class InstructionCost;
struct VPCostContext;
+/// Controls how register usage is estimated for a VPlan.
+enum class VPRegisterUsageMode { LiveIntervals, ConservativePeak };
+
// Collect a VPlan's ephemeral recipes (those used only by an assume).
void collectEphemeralRecipesForVPlan(VPlan &Plan,
DenseSet<VPRecipeBase *> &EphRecipes);
@@ -40,6 +43,12 @@ struct VPRegisterUsage {
/// The key is ClassID of target-provided register class.
SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
+ /// Check if the estimated pressure for register class \p ClassID exceeds
+ /// the number of available registers for the target. If non-zero,
+ /// OverrideMaxNumRegs is used in place of the target's number of registers.
+ bool exceedsMaxNumRegs(unsigned ClassID, const TargetTransformInfo &TTI,
+ unsigned OverrideMaxNumRegs = 0) const;
+
/// Calculate the estimated cost of any spills due to using more registers
/// than the number available for the target. If non-zero, OverrideMaxNumRegs
/// is used in place of the target's number of registers.
@@ -51,10 +60,11 @@ struct VPRegisterUsage {
/// Estimate the register usage for \p Plan and vectorization factors in \p VFs
/// by calculating the highest number of values that are live at a single
/// location as a rough estimate. Returns the register usage for each VF in \p
-/// VFs.
-SmallVector<VPRegisterUsage, 8>
-calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef<ElementCount> VFs,
- const TargetTransformInfo &TTI);
+/// VFs. Conservative mode also accounts for target-specific result/operand
+/// register-reuse constraints at operation definitions.
+LLVM_ABI_FOR_TEST SmallVector<VPRegisterUsage, 8> calculateRegisterUsageForPlan(
+ VPlan &Plan, ArrayRef<ElementCount> VFs, const TargetTransformInfo &TTI,
+ VPRegisterUsageMode Mode = VPRegisterUsageMode::LiveIntervals);
} // end namespace llvm
diff --git a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
index 712dad31296a3..17b6cecbcddfd 100644
--- a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
@@ -10,6 +10,7 @@ set(LLVM_LINK_COMPONENTS
add_llvm_unittest(VectorizeTests
VPlanTest.cpp
+ VPlanAnalysisTest.cpp
VPDomTreeTest.cpp
VPPostDomFrontierTest.cpp
VPlanHCFGTest.cpp
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanAnalysisTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanAnalysisTest.cpp
new file mode 100644
index 0000000000000..5406ab3421bda
--- /dev/null
+++ b/llvm/unittests/Transforms/Vectorize/VPlanAnalysisTest.cpp
@@ -0,0 +1,233 @@
+//===- llvm/unittests/Transforms/Vectorize/VPlanAnalysisTest.cpp ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../lib/Transforms/Vectorize/VPlanAnalysis.h"
+#include "VPlanTestBase.h"
+#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/Analysis/TargetTransformInfoImpl.h"
+#include "llvm/IR/Instruction.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+class MockRegPressureTTI
+ : public TargetTransformInfoImplCRTPBase<MockRegPressureTTI> {
+ bool ModelResultPressure = false;
+ bool CheckLiveInCastOperand = false;
+ mutable bool SawLiveInCastOperand = false;
+ SmallBitVector ReusableOperandsMask;
+
+public:
+ explicit MockRegPressureTTI(const DataLayout &DL)
+ : TargetTransformInfoImplCRTPBase<MockRegPressureTTI>(DL) {}
+
+ void setModelResultPressure(bool B) { ModelResultPressure = B; }
+ void checkLiveInCastOperand() { CheckLiveInCastOperand = true; }
+ bool sawLiveInCastOperand() const { return SawLiveInCastOperand; }
+ void setReusableOperandsMask(SmallBitVector Mask) {
+ ReusableOperandsMask = std::move(Mask);
+ }
+
+ unsigned getNumberOfRegisters(unsigned ClassID) const {
+ (void)ClassID;
+ return 32;
+ }
+
+ unsigned getRegisterClassForType(bool Vector, Type *Ty) const {
+ (void)Vector;
+ (void)Ty;
+ return 1;
+ }
+
+ unsigned getRegUsageForType(Type *Ty) const {
+ (void)Ty;
+ return 1;
+ }
+
+ std::optional<SmallBitVector> getResultRegisterReuseMask(
+ unsigned Opcode, Type *ResultType,
+ ArrayRef<TTI::RegisterUsageOperandInfo> Operands) const {
+ if (!ModelResultPressure)
+ return std::nullopt;
+ if (Opcode != Instruction::Add)
+ return std::nullopt;
+
+ EXPECT_TRUE(ResultType->isVectorTy());
+ EXPECT_EQ(Operands.size(), 2u);
+ if (CheckLiveInCastOperand && Operands.size() == 2) {
+ LLVMContext &Ctx = ResultType->getContext();
+ EXPECT_EQ(Operands[1].ValueType,
+ FixedVectorType::get(Type::getInt32Ty(Ctx), 4));
+ EXPECT_EQ(Operands[1].SourceType,
+ FixedVectorType::get(Type::getInt16Ty(Ctx), 4));
+ EXPECT_EQ(Operands[1].DefOpcode, Instruction::SExt);
+ EXPECT_TRUE(Operands[1].IsUniform);
+ SawLiveInCastOperand = true;
+ }
+
+ SmallBitVector ReusableOperands = ReusableOperandsMask;
+ ReusableOperands.resize(Operands.size());
+ return ReusableOperands;
+ }
+};
+
+class VPRegisterUsageAnalysisTest : public VPlanTestBase {
+protected:
+ VPlan &buildSingleAddPlan(bool KeepFirstOperandLive = false,
+ bool RepeatFirstOperand = false,
+ bool UseLiveInCast = false) {
+ VPlan &Plan = getPlan();
+ auto *I32Ty = Type::getInt32Ty(C);
+
+ VPBasicBlock *Preheader = Plan.getEntry();
+ VPBasicBlock *Header = Plan.createVPBasicBlock("header");
+ VPBasicBlock *Latch = Plan.createVPBasicBlock("latch");
+ auto *LoopRegion = Plan.createLoopRegion(I32Ty, DebugLoc::getUnknown(),
+ "vector.loop", Header, Latch);
+
+ auto *LiveInA = Plan.getConstantInt(I32Ty, 1);
+ auto *LiveInB = Plan.getConstantInt(I32Ty, 2);
+ auto *Seed0 =
+ new VPWidenRecipe(Instruction::Mul, {LiveInA, LiveInB},
+ VPIRFlags::getDefaultFlags(Instruction::Mul));
+ auto *Seed1 =
+ new VPWidenRecipe(Instruction::Sub, {LiveInA, LiveInB},
+ VPIRFlags::getDefaultFlags(Instruction::Sub));
+ Header->appendRecipe(Seed0);
+ Header->appendRecipe(Seed1);
+
+ VPValue *SecondAddOperand = Seed1;
+ if (RepeatFirstOperand)
+ SecondAddOperand = Seed0;
+ if (UseLiveInCast) {
+ auto *Ext =
+ new SExtInst(ConstantInt::get(Type::getInt16Ty(C), 1), I32Ty, "ext",
+ ScalarHeader->getTerminator()->getIterator());
+ SecondAddOperand = Plan.getOrAddLiveIn(Ext);
+ }
+ auto *Add = new VPInstruction(Instruction::Add, {Seed0, SecondAddOperand},
+ VPIRFlags::getDefaultFlags(Instruction::Add));
+ Header->appendRecipe(Add);
+ VPValue *SecondConsumeOperand =
+ KeepFirstOperandLive ? static_cast<VPValue *>(Seed0)
+ : RepeatFirstOperand ? static_cast<VPValue *>(Seed1)
+ : static_cast<VPValue *>(LiveInA);
+ auto *Consume =
+ new VPWidenRecipe(Instruction::Sub, {Add, SecondConsumeOperand},
+ VPIRFlags::getDefaultFlags(Instruction::Sub));
+ Latch->appendRecipe(Consume);
+ Latch->appendRecipe(
+ new VPInstruction(VPInstruction::BranchOnCond, {Plan.getTrue()}));
+
+ VPBlockUtils::connectBlocks(Header, Latch);
+ VPBlockUtils::connectBlocks(Preheader, LoopRegion);
+ VPBlockUtils::connectBlocks(LoopRegion, Plan.getScalarHeader());
+ return Plan;
+ }
+
+ unsigned getMaxLocalUsers(VPlan &Plan, TargetTransformInfo &TTI,
+ VPRegisterUsageMode Mode) {
+ SmallVector<VPRegisterUsage, 1> Usage = calculateRegisterUsageForPlan(
+ Plan, {ElementCount::getFixed(4)}, TTI, Mode);
+ EXPECT_EQ(Usage.size(), 1u);
+ return Usage[0].MaxLocalUsers.lookup(1);
+ }
+};
+
+TEST_F(VPRegisterUsageAnalysisTest, DefaultPressureDoesNotCountResultAtDef) {
+ VPlan &Plan = buildSingleAddPlan();
+ auto Impl = std::make_unique<MockRegPressureTTI>(Plan.getDataLayout());
+ Impl->setModelResultPressure(true);
+ auto TTI = TargetTransformInfo(std::move(Impl));
+
+ auto Usage =
+ calculateRegisterUsageForPlan(Plan, {ElementCount::getFixed(4)}, TTI);
+ ASSERT_EQ(Usage.size(), 1u);
+ EXPECT_EQ(Usage[0].MaxLocalUsers.lookup(1), 2u);
+ EXPECT_EQ(getMaxLocalUsers(Plan, TTI, VPRegisterUsageMode::LiveIntervals),
+ 2u);
+ EXPECT_EQ(getMaxLocalUsers(Plan, TTI, VPRegisterUsageMode::ConservativePeak),
+ 3u);
+}
+
+TEST_F(VPRegisterUsageAnalysisTest,
+ ConservativePressureCountsResultWithoutReusableDyingOperand) {
+ VPlan &Plan = buildSingleAddPlan();
+ auto Impl = std::make_unique<MockRegPressureTTI>(Plan.getDataLayout());
+ Impl->setModelResultPressure(true);
+ auto TTI = TargetTransformInfo(std::move(Impl));
+
+ EXPECT_EQ(getMaxLocalUsers(Plan, TTI, VPRegisterUsageMode::ConservativePeak),
+ 3u);
+}
+
+TEST_F(VPRegisterUsageAnalysisTest,
+ ConservativePressureKeepsPeakAtTwoWithReusableDyingOperand) {
+ VPlan &Plan = buildSingleAddPlan();
+
+ auto Impl = std::make_unique<MockRegPressureTTI>(Plan.getDataLayout());
+ Impl->setModelResultPressure(true);
+ SmallBitVector ReusableOperands(2, false);
+ ReusableOperands.set(0);
+ Impl->setReusableOperandsMask(std::move(ReusableOperands));
+ auto TTI = TargetTransformInfo(std::move(Impl));
+
+ EXPECT_EQ(getMaxLocalUsers(Plan, TTI, VPRegisterUsageMode::ConservativePeak),
+ 2u);
+}
+
+TEST_F(VPRegisterUsageAnalysisTest,
+ ConservativePressureDoesNotReuseLiveOperand) {
+ VPlan &Plan = buildSingleAddPlan(/*KeepFirstOperandLive=*/true);
+
+ auto Impl = std::make_unique<MockRegPressureTTI>(Plan.getDataLayout());
+ Impl->setModelResultPressure(true);
+ SmallBitVector ReusableOperands(2, false);
+ ReusableOperands.set(0);
+ Impl->setReusableOperandsMask(std::move(ReusableOperands));
+ auto TTI = TargetTransformInfo(std::move(Impl));
+
+ EXPECT_EQ(getMaxLocalUsers(Plan, TTI, VPRegisterUsageMode::ConservativePeak),
+ 3u);
+}
+
+TEST_F(VPRegisterUsageAnalysisTest,
+ ConservativePressureRequiresAllDuplicateUsesToBeReusable) {
+ VPlan &Plan = buildSingleAddPlan(/*KeepFirstOperandLive=*/false,
+ /*RepeatFirstOperand=*/true);
+
+ auto Impl = std::make_unique<MockRegPressureTTI>(Plan.getDataLayout());
+ Impl->setModelResultPressure(true);
+ SmallBitVector ReusableOperands(2, false);
+ ReusableOperands.set(0);
+ Impl->setReusableOperandsMask(std::move(ReusableOperands));
+ auto TTI = TargetTransformInfo(std::move(Impl));
+
+ EXPECT_EQ(getMaxLocalUsers(Plan, TTI, VPRegisterUsageMode::ConservativePeak),
+ 3u);
+}
+
+TEST_F(VPRegisterUsageAnalysisTest,
+ ConservativePressureDescribesLiveInCastOperand) {
+ VPlan &Plan = buildSingleAddPlan(/*KeepFirstOperandLive=*/false,
+ /*RepeatFirstOperand=*/false,
+ /*UseLiveInCast=*/true);
+
+ auto Impl = std::make_unique<MockRegPressureTTI>(Plan.getDataLayout());
+ Impl->setModelResultPressure(true);
+ Impl->checkLiveInCastOperand();
+ MockRegPressureTTI *ImplPtr = Impl.get();
+ auto TTI = TargetTransformInfo(std::move(Impl));
+
+ (void)getMaxLocalUsers(Plan, TTI, VPRegisterUsageMode::ConservativePeak);
+ EXPECT_TRUE(ImplPtr->sawLiveInCastOperand());
+}
+
+} // namespace
>From 17568c9cbefdbe9628f5327461f4a23be9e1a6a2 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Mon, 7 Sep 2026 17:04:42 +0800
Subject: [PATCH 3/3] [LV][RISCV] Prune max-bandwidth VFs by register pressure
RISC-V's shouldMaximizeVectorBandwidth policy considers fixed and scalable VFs beyond the ordinary widest-type limit. These candidates can exceed the available vector registers when predicates and widening operations are present.
Use VPlan's conservative register-usage mode for those automatically expanded candidates when the target also enables vectorization register-pressure handling. Model RVV widening add/sub result reuse for both fixed and scalable vectors, including operand liveness, uniform VX/WX forms, and early-clobber constraints, while keeping the normal spill-cost path on LiveIntervals.
Forced VFs, disabled max-bandwidth or register-pressure policies, and other targets retain their existing behavior.
Assisted-by: TRAE CLI (GPT-5)
---
.../llvm/Analysis/TargetTransformInfo.h | 4 +-
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 79 +++++++-
.../Target/RISCV/RISCVTargetTransformInfo.h | 4 +
.../Vectorize/LoopVectorizationPlanner.cpp | 20 +-
.../Vectorize/LoopVectorizationPlanner.h | 4 +
.../Transforms/Vectorize/LoopVectorize.cpp | 15 ++
.../RISCV/partial-reduce-with-predicate.ll | 2 +-
.../RISCV/reg-usage-maxbandwidth.ll | 110 ++++++++++-
.../reg-usage-widening-mixed-source-widths.ll | 48 +++++
.../RISCV/reg-usage-widening-multi-user.ll | 181 ++++++++++++++++++
.../RISCV/reg-usage-widening-pressure.ll | 98 ++++++++++
.../RISCV/reg-usage-widening-same-value.ll | 58 ++++++
12 files changed, 609 insertions(+), 14 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-mixed-source-widths.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-multi-user.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-pressure.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-same-value.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 93c6185a8fdeb..7d72365f8059d 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1998,8 +1998,8 @@ class TargetTransformInfo {
/// processing \p Iters scalar iterations per vector iteration.
LLVM_ABI bool preferEpilogueVectorization(ElementCount Iters) const;
- /// \returns True if the loop vectorizer should discard any VFs where the
- /// maximum register pressure exceeds getNumberOfRegisters.
+ /// \returns True if the loop vectorizer should account for register pressure
+ /// when selecting VFs.
LLVM_ABI bool shouldConsiderVectorizationRegPressure() const;
/// \returns True if the target wants to expand the given reduction intrinsic
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 62ba4e92be257..0e081b35a5195 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -484,7 +484,7 @@ bool RISCVTTIImpl::shouldMaximizeVectorBandwidth(
// Size vector factors by the smallest type in the loop rather than the
// widest, so loops mixing narrow and wide types can use the full vector
// register bandwidth. Overly wide factors are pruned separately by register
- // pressure estimation (shouldConsiderVectorizationRegPressure).
+ // pressure estimation.
return K != TargetTransformInfo::RGK_Scalar && ST->hasVInstructions();
}
@@ -3429,6 +3429,83 @@ unsigned RISCVTTIImpl::getRegUsageForType(Type *Ty) const {
return BaseT::getRegUsageForType(Ty);
}
+std::optional<SmallBitVector> RISCVTTIImpl::getResultRegisterReuseMask(
+ unsigned Opcode, Type *ResultType,
+ ArrayRef<TTI::RegisterUsageOperandInfo> Operands) const {
+ bool UsesRVV = isa<ScalableVectorType>(ResultType)
+ ? ST->hasVInstructions()
+ : isa<FixedVectorType>(ResultType) &&
+ ST->useRVVForFixedLengthVectors();
+ if (!UsesRVV || !ResultType->getScalarType()->isIntegerTy())
+ return std::nullopt;
+ auto ResultLT = getTypeLegalizationCost(ResultType);
+ if (ResultLT.first != 1 || !ResultLT.second.isVector())
+ return std::nullopt;
+
+ auto IsWideningTypePair = [this](Type *SrcType, Type *DstType) {
+ if (!SrcType || !DstType || !isa<VectorType>(SrcType) ||
+ !isa<VectorType>(DstType) || !SrcType->getScalarType()->isIntegerTy() ||
+ !DstType->getScalarType()->isIntegerTy())
+ return false;
+ if (cast<VectorType>(SrcType)->getElementCount() !=
+ cast<VectorType>(DstType)->getElementCount())
+ return false;
+
+ unsigned SrcWidth = SrcType->getScalarType()->getIntegerBitWidth();
+ unsigned DstWidth = DstType->getScalarType()->getIntegerBitWidth();
+ if (SrcWidth < 8 || DstWidth > ST->getELen() || SrcWidth >= DstWidth ||
+ DstWidth % SrcWidth != 0 || !isPowerOf2_32(DstWidth / SrcWidth))
+ return false;
+
+ auto SrcLT = getTypeLegalizationCost(SrcType);
+ auto *IntermediateType =
+ VectorType::get(IntegerType::get(DstType->getContext(), DstWidth / 2),
+ cast<VectorType>(DstType));
+ auto IntermediateLT = getTypeLegalizationCost(IntermediateType);
+ return SrcLT.first == 1 && SrcLT.second.isVector() &&
+ IntermediateLT.first == 1 && IntermediateLT.second.isVector();
+ };
+
+ if ((Opcode != Instruction::Add && Opcode != Instruction::Sub) ||
+ Operands.size() != 2)
+ return std::nullopt;
+
+ auto IsWideningExtend = [ResultType, &IsWideningTypePair](
+ const TTI::RegisterUsageOperandInfo &Op) {
+ if (Op.ValueType != ResultType || (Op.DefOpcode != Instruction::SExt &&
+ Op.DefOpcode != Instruction::ZExt))
+ return false;
+ return IsWideningTypePair(Op.SourceType, ResultType);
+ };
+
+ bool Ext0 = IsWideningExtend(Operands[0]);
+ bool Ext1 = IsWideningExtend(Operands[1]);
+ // VV/VX forms and WX forms use an early-clobber result that cannot reuse an
+ // operand vector register. A uniform extending operand lowers through a
+ // scalar register and selects the VX/WX form.
+ if ((Ext0 && Ext1) || (Ext0 && Operands[0].IsUniform) ||
+ (Ext1 && Operands[1].IsUniform))
+ return SmallBitVector(Operands.size());
+
+ // Widening add is commutative, while widening sub requires the extended
+ // narrow operand to be on the right. The wide operand may be tied to the
+ // result; VPlan determines whether its live range ends at this operation.
+ if (Opcode == Instruction::Add && Ext0 != Ext1) {
+ unsigned WideOpIdx = Ext0 ? 1 : 0;
+ SmallBitVector ReusableOperands(Operands.size());
+ if (!Operands[WideOpIdx].IsUniform)
+ ReusableOperands.set(WideOpIdx);
+ return ReusableOperands;
+ }
+ if (Opcode == Instruction::Sub && Ext1 && !Ext0) {
+ SmallBitVector ReusableOperands(Operands.size());
+ if (!Operands[0].IsUniform)
+ ReusableOperands.set(0);
+ return ReusableOperands;
+ }
+ return std::nullopt;
+}
+
unsigned RISCVTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
if (SLPMaxVF.getNumOccurrences())
return SLPMaxVF;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 67ea98c3f5a72..7166b0465901a 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -139,6 +139,10 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
unsigned getRegUsageForType(Type *Ty) const override;
+ std::optional<SmallBitVector> getResultRegisterReuseMask(
+ unsigned Opcode, Type *ResultType,
+ ArrayRef<TTI::RegisterUsageOperandInfo> Operands) const override;
+
unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const override;
bool preferAlternateOpcodeVectorization() const override;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index c464c7894ae5c..64e0d486c9968 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -43,7 +43,7 @@ static cl::opt<bool> UseWiderVFIfCallVariantsPresent(
static cl::opt<bool> ConsiderRegPressure(
"vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden,
- cl::desc("Discard VFs if their register pressure is too high."));
+ cl::desc("Consider register pressure when selecting VFs."));
static cl::opt<bool> ForceTargetSupportsScalableVectors(
"force-target-supports-scalable-vectors", cl::init(false), cl::Hidden,
@@ -194,6 +194,23 @@ bool VFSelectionContext::shouldConsiderRegPressureForVF(ElementCount VF) const {
: MaxPermissibleVFWithoutMaxBW.FixedVF);
}
+bool VFSelectionContext::shouldDiscardMaxBandwidthVFForRegPressure(
+ ElementCount VF) const {
+ if (ConsiderRegPressure.getNumOccurrences() && !ConsiderRegPressure)
+ return false;
+
+ auto RegKind = VF.isScalable() ? TargetTransformInfo::RGK_ScalableVector
+ : TargetTransformInfo::RGK_FixedWidthVector;
+ ElementCount MaxVFWithoutMaxBW = VF.isScalable()
+ ? MaxPermissibleVFWithoutMaxBW.ScalableVF
+ : MaxPermissibleVFWithoutMaxBW.FixedVF;
+ return VF.isVector() && useMaxBandwidth(VF.isScalable()) &&
+ TTI.shouldMaximizeVectorBandwidth(RegKind) &&
+ TTI.shouldConsiderVectorizationRegPressure() &&
+ MaxVFWithoutMaxBW.isVector() &&
+ ElementCount::isKnownGT(VF, MaxVFWithoutMaxBW);
+}
+
ElementCount VFSelectionContext::clampVFByMaxTripCount(
ElementCount VF, unsigned MaxTripCount, unsigned UserIC,
bool FoldTailByMasking, bool RequiresScalarEpilogue) const {
@@ -396,6 +413,7 @@ VFSelectionContext::getMaxLegalScalableVF(unsigned MaxSafeElements) {
FixedScalableVFPair VFSelectionContext::computeFeasibleMaxVF(
unsigned MaxTripCount, ElementCount UserVF, unsigned UserIC,
bool FoldTailByMasking, bool RequiresScalarEpilogue) {
+ MaxPermissibleVFWithoutMaxBW = FixedScalableVFPair::getNone();
auto [SmallestType, WidestType] = getSmallestAndWidestTypes();
// Get the maximum safe dependence distance in bits computed by LAA.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 2e5b0f64bfed2..2c703ffcb67b8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -762,6 +762,10 @@ class VFSelectionContext {
/// \return True if register pressure should be considered for the given VF.
bool shouldConsiderRegPressureForVF(ElementCount VF) const;
+ /// \return True if \p VF is a max-bandwidth candidate that should be
+ /// rejected when it exceeds the target's available registers.
+ bool shouldDiscardMaxBandwidthVFForRegPressure(ElementCount VF) const;
+
/// \return True if scalable vectors are supported by the target or forced.
bool supportsScalableVectors() const;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d71369e73b002..35ba5e4c66b63 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5798,11 +5798,18 @@ LoopVectorizationPlanner::computeBestVF() {
P->vectorFactors().end());
SmallVector<VPRegisterUsage, 8> RUs;
+ SmallVector<VPRegisterUsage, 8> MaxBandwidthRUs;
bool ConsiderRegPressure = any_of(VFs, [this](ElementCount VF) {
return Config.shouldConsiderRegPressureForVF(VF);
});
+ bool PruneMaxBandwidthVFs = any_of(VFs, [this](ElementCount VF) {
+ return Config.shouldDiscardMaxBandwidthVFForRegPressure(VF);
+ });
if (ConsiderRegPressure)
RUs = calculateRegisterUsageForPlan(*P, VFs, TTI);
+ if (PruneMaxBandwidthVFs)
+ MaxBandwidthRUs = calculateRegisterUsageForPlan(
+ *P, VFs, TTI, VPRegisterUsageMode::ConservativePeak);
for (unsigned I = 0; I < VFs.size(); I++) {
ElementCount VF = VFs[I];
@@ -5823,6 +5830,14 @@ LoopVectorizationPlanner::computeBestVF() {
<< " which isn't allowed when optimizing for size.\n");
continue;
}
+ if (Config.shouldDiscardMaxBandwidthVFForRegPressure(VF) &&
+ MaxBandwidthRUs[I].exceedsMaxNumRegs(
+ TTI.getRegisterClassForType(/*Vector=*/true), TTI,
+ ForceTargetNumVectorRegs)) {
+ LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width "
+ << VF << " because it uses too many registers\n");
+ continue;
+ }
InstructionCost Cost =
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
index 8c94187b0c786..5e05750bdf4fe 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
-; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+experimental-zvdot4a8i -S < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+experimental-zvdot4a8i -force-vector-width='vscale x 16' -S < %s | FileCheck %s --check-prefixes=CHECK
; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+experimental-zvdot4a8i -scalable-vectorization=off -tail-folding-policy=dont-fold-tail -S < %s | FileCheck %s --check-prefixes=CHECK-FIXED
define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll
index 41f1a9b45b7ff..5b265013cc8c7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll
@@ -1,15 +1,54 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV\(REG\): (?!At)" --filter "LV: Selecting VF" --version 6
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV\(REG\): (Cost of|Not considering vector loop)" --filter "Cost for VF (16|32):" --filter "LV: Selecting VF" --filter "Executing best plan with VF=(vscale x 16|32), UF=1" --version 6
; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -vectorizer-maximize-bandwidth -debug-only=loop-vectorize,vplan -disable-output -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-REGS-VP
+; RUN: llvm-extract -S --func=dotp %s -o - | opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -force-target-num-vector-regs=16 -force-vector-interleave=1 -debug-only=loop-vectorize,vplan -disable-output 2>&1 | FileCheck %s --check-prefix=POLICY
+; RUN: llvm-extract -S --func=dotp %s -o - | opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -force-target-num-vector-regs=5 -force-vector-interleave=1 -debug-only=loop-vectorize,vplan -disable-output 2>&1 | FileCheck %s --check-prefix=SOFT-HARD --implicit-check-not='LV(REG): Not considering vector loop of width vscale x 4'
+; RUN: llvm-extract -S --func=dotp %s -o - | opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -force-target-num-vector-regs=16 -force-vector-width='vscale x 16' -force-vector-interleave=1 -debug-only=loop-vectorize,vplan -disable-output 2>&1 | FileCheck %s --check-prefix=FORCED --implicit-check-not='LV(REG): Not considering vector loop of width vscale x 16'
+; RUN: llvm-extract -S --func=dotp %s -o - | opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -vectorizer-maximize-bandwidth=false -force-target-num-vector-regs=1 -force-vector-interleave=1 -debug-only=loop-vectorize,vplan -disable-output 2>&1 | FileCheck %s --check-prefix=NOMAXBW --implicit-check-not='LV(REG): Not considering vector loop'
+; RUN: llvm-extract -S --func=dotp %s -o - | opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -vectorizer-consider-reg-pressure=false -force-target-num-vector-regs=16 -force-vector-interleave=1 -debug-only=loop-vectorize,vplan -disable-output 2>&1 | FileCheck %s --check-prefix=NOPRESSURE --implicit-check-not='LV(REG): Not considering vector loop'
+; RUN: llvm-extract -S --func=dotp %s -o - | opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -scalable-vectorization=off -force-target-num-vector-regs=16 -force-vector-interleave=1 -debug-only=loop-vectorize,vplan -disable-output 2>&1 | FileCheck %s --check-prefix=FIXEDPOLICY
+; RUN: llvm-extract -S --func=dotp %s -o - | opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -scalable-vectorization=off -force-vector-width=32 -force-target-num-vector-regs=16 -force-vector-interleave=1 -debug-only=loop-vectorize,vplan -disable-output 2>&1 | FileCheck %s --check-prefix=FORCED-FIXED --implicit-check-not='LV(REG): Not considering vector loop'
+; RUN: llvm-extract -S --func=invariant_vector_pressure %s -o - | opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -force-target-num-vector-regs=16 -force-vector-interleave=1 -debug-only=loop-vectorize,vplan -disable-output 2>&1 | FileCheck %s --check-prefix=INVARIANT
+; RUN: llvm-extract -S --func=vector_class_only %s -o - | opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -force-target-num-vector-regs=4 -force-vector-interleave=1 -debug-only=loop-vectorize,vplan -disable-output 2>&1 | FileCheck %s --check-prefix=VECTORCLASS --implicit-check-not='Not considering vector loop of width vscale x 4'
define i32 @dotp(ptr %a, ptr %b) {
-; CHECK-REGS-VP: LV(REG): VF = vscale x 16
-; CHECK-REGS-VP-NEXT: LV(REG): Found max usage: 2 item
-; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
-; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
-; CHECK-REGS-VP-NEXT: LV(REG): Found invariant usage: 1 item
-; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
-; CHECK-REGS-VP: LV: Selecting VF: vscale x 16.
+; POLICY-LABEL: 'dotp'
+; POLICY: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; POLICY: LV: Selecting VF: vscale x 8.
+;
+; SOFT-HARD-LABEL: 'dotp'
+; SOFT-HARD: LV(REG): Cost of 2 from 1 spills of RISCV::GPRRC
+; SOFT-HARD: LV(REG): Cost of 2 from 1 spills of RISCV::GPRRC
+; SOFT-HARD: LV(REG): Cost of 2 from 1 spills of RISCV::GPRRC
+; SOFT-HARD: LV(REG): Cost of 2 from 1 spills of RISCV::VRRC
+; SOFT-HARD: LV(REG): Not considering vector loop of width vscale x 8 because it uses too many registers
+; SOFT-HARD: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; SOFT-HARD: LV: Selecting VF: vscale x 4.
+;
+; FORCED-LABEL: 'dotp'
+; FORCED: Executing best plan with VF=vscale x 16, UF=1
+;
+; NOMAXBW-LABEL: 'dotp'
+; NOMAXBW: LV(REG): Cost of 10 from 5 spills of RISCV::GPRRC
+; NOMAXBW: LV(REG): Cost of 4 from 2 spills of RISCV::VRRC
+; NOMAXBW: LV(REG): Cost of 10 from 5 spills of RISCV::GPRRC
+; NOMAXBW: LV(REG): Cost of 4 from 2 spills of RISCV::VRRC
+; NOMAXBW: LV(REG): Cost of 10 from 5 spills of RISCV::GPRRC
+; NOMAXBW: LV(REG): Cost of 10 from 5 spills of RISCV::VRRC
+; NOMAXBW: LV: Selecting VF: vscale x 4.
+;
+; NOPRESSURE-LABEL: 'dotp'
+; NOPRESSURE: LV: Selecting VF: vscale x 16.
+; NOPRESSURE: Executing best plan with VF=vscale x 16, UF=1
+;
+; FIXEDPOLICY-LABEL: 'dotp'
+; FIXEDPOLICY: Cost for VF 16: 16 (Estimated cost per lane: 1)
+; FIXEDPOLICY: LV(REG): Not considering vector loop of width 32 because it uses too many registers
+; FIXEDPOLICY: LV: Selecting VF: 16.
+;
+; FORCED-FIXED-LABEL: 'dotp'
+; FORCED-FIXED: Cost for VF 32: 30 (Estimated cost per lane: 0.937)
+; FORCED-FIXED: Executing best plan with VF=32, UF=1
+;
entry:
br label %for.body
@@ -32,3 +71,56 @@ for.body:
for.exit:
ret i32 %add
}
+
+define void @invariant_vector_pressure(ptr noalias %src, ptr noalias %dst,
+; INVARIANT-LABEL: 'invariant_vector_pressure'
+; INVARIANT: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; INVARIANT: LV: Selecting VF: vscale x 8.
+;
+ ptr %bias.ptr, i64 %n) {
+entry:
+ %bias = load i32, ptr %bias.ptr, align 4
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %src.gep = getelementptr i8, ptr %src, i64 %iv
+ %x = load i8, ptr %src.gep, align 1
+ %ext = zext i8 %x to i32
+ %sum = add i32 %ext, %bias
+ %dst.gep = getelementptr i32, ptr %dst, i64 %iv
+ store i32 %sum, ptr %dst.gep, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @vector_class_only(ptr noalias %src, ptr noalias %dst, i64 %n) {
+; VECTORCLASS-LABEL: 'vector_class_only'
+; VECTORCLASS: LV(REG): Cost of 4 from 2 spills of RISCV::GPRRC
+; VECTORCLASS: LV(REG): Cost of 4 from 2 spills of RISCV::GPRRC
+; VECTORCLASS: LV(REG): Cost of 4 from 2 spills of RISCV::GPRRC
+; VECTORCLASS: LV(REG): Not considering vector loop of width vscale x 8 because it uses too many registers
+; VECTORCLASS: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; VECTORCLASS: LV: Selecting VF: vscale x 4.
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %src.gep = getelementptr i8, ptr %src, i64 %iv
+ %x = load i8, ptr %src.gep, align 1
+ %ext = zext i8 %x to i64
+ %dst.gep = getelementptr i64, ptr %dst, i64 %iv
+ store i64 %ext, ptr %dst.gep, align 8
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-mixed-source-widths.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-mixed-source-widths.ll
new file mode 100644
index 0000000000000..d3ae4f51d6856
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-mixed-source-widths.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: (Found feasible scalable VF|Selecting VF)" --filter "LV\(REG\): (Calculating conservative max register usage|VF = vscale x (4|16)$|RegisterClass: RISCV::VRRC, (4|6|16|24) registers)" --version 6
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -vectorizer-maximize-bandwidth -force-vector-interleave=1 -force-target-num-vector-regs=64 -debug-only=loop-vectorize,vplan -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PRESSURE
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -vectorizer-maximize-bandwidth=false -force-vector-interleave=1 -force-target-num-vector-regs=4 -debug-only=loop-vectorize,vplan -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SOFT --implicit-check-not='LV(REG): Calculating conservative max register usage:' --implicit-check-not='spills of RISCV::VRRC'
+
+define void @mixed_source_width_same_extension(ptr noalias %src8,
+; PRESSURE-LABEL: 'mixed_source_width_same_extension'
+; PRESSURE: LV: Found feasible scalable VF = vscale x 16
+; PRESSURE: LV(REG): VF = vscale x 4
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 4 registers
+; PRESSURE: LV(REG): VF = vscale x 16
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 16 registers
+; PRESSURE: LV(REG): Calculating conservative max register usage:
+; PRESSURE: LV(REG): VF = vscale x 4
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 6 registers
+; PRESSURE: LV(REG): VF = vscale x 16
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
+; PRESSURE: LV: Selecting VF: vscale x 16.
+;
+; SOFT-LABEL: 'mixed_source_width_same_extension'
+; SOFT: LV: Found feasible scalable VF = vscale x 4
+; SOFT: LV(REG): VF = vscale x 4
+; SOFT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers
+; SOFT: LV: Selecting VF: vscale x 4.
+;
+ ptr noalias %src16,
+ ptr noalias %dst, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %p8 = getelementptr i8, ptr %src8, i64 %iv
+ %a = load i8, ptr %p8, align 1
+ %p16 = getelementptr i16, ptr %src16, i64 %iv
+ %b = load i16, ptr %p16, align 2
+ %ae = sext i8 %a to i32
+ %be = sext i16 %b to i32
+ %sum = add i32 %ae, %be
+ %out = getelementptr i32, ptr %dst, i64 %iv
+ store i32 %sum, ptr %out, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-multi-user.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-multi-user.ll
new file mode 100644
index 0000000000000..1cd17e6169096
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-multi-user.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV\(REG\): (Calculating conservative max register usage|VF = vscale x 8$|RegisterClass: RISCV::VRRC, (6|8|10|12|16) registers)" --version 6
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -vectorizer-maximize-bandwidth -force-vector-interleave=1 -force-target-num-vector-regs=64 -debug-only=loop-vectorize,vplan -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PRESSURE
+
+define void @multi_user_widening(ptr noalias %src16, ptr noalias %wide0,
+; PRESSURE-LABEL: 'multi_user_widening'
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 6 registers
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 12 registers
+; PRESSURE: LV(REG): Calculating conservative max register usage:
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 8 registers
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 16 registers
+;
+ ptr noalias %wide1, ptr noalias %out0,
+ ptr noalias %out1, ptr noalias %out2, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %src.gep = getelementptr i16, ptr %src16, i64 %iv
+ %x = load i16, ptr %src.gep, align 2
+ %x.ext = sext i16 %x to i32
+ %wide0.gep = getelementptr i32, ptr %wide0, i64 %iv
+ %a = load i32, ptr %wide0.gep, align 4
+ %wide1.gep = getelementptr i32, ptr %wide1, i64 %iv
+ %b = load i32, ptr %wide1.gep, align 4
+ %r0 = add i32 %a, %x.ext
+ %r1 = add i32 %b, %x.ext
+ %out0.gep = getelementptr i32, ptr %out0, i64 %iv
+ %out1.gep = getelementptr i32, ptr %out1, i64 %iv
+ %out2.gep = getelementptr i32, ptr %out2, i64 %iv
+ store i32 %r0, ptr %out0.gep, align 4
+ store i32 %r1, ptr %out1.gep, align 4
+ store i32 %b, ptr %out2.gep, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @multi_user_widening_sub(ptr noalias %src16, ptr noalias %wide,
+; PRESSURE-LABEL: 'multi_user_widening_sub'
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 8 registers
+; PRESSURE: LV(REG): Calculating conservative max register usage:
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 6 registers
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 12 registers
+;
+ ptr noalias %out0, ptr noalias %out1,
+ i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %src.gep = getelementptr i16, ptr %src16, i64 %iv
+ %x = load i16, ptr %src.gep, align 2
+ %x.ext = sext i16 %x to i32
+ %wide.gep = getelementptr i32, ptr %wide, i64 %iv
+ %a = load i32, ptr %wide.gep, align 4
+ %r = sub i32 %a, %x.ext
+ %out0.gep = getelementptr i32, ptr %out0, i64 %iv
+ %out1.gep = getelementptr i32, ptr %out1, i64 %iv
+ store i32 %r, ptr %out0.gep, align 4
+ store i32 %a, ptr %out1.gep, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @last_use_widening_sub(ptr noalias %src16, ptr noalias %wide,
+; PRESSURE-LABEL: 'last_use_widening_sub'
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 8 registers
+; PRESSURE: LV(REG): Calculating conservative max register usage:
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 8 registers
+;
+ ptr noalias %out, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %src.gep = getelementptr i16, ptr %src16, i64 %iv
+ %x = load i16, ptr %src.gep, align 2
+ %x.ext = sext i16 %x to i32
+ %wide.gep = getelementptr i32, ptr %wide, i64 %iv
+ %a = load i32, ptr %wide.gep, align 4
+ %r = sub i32 %a, %x.ext
+ %out.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %out.gep, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @uniform_narrow_widening_add(ptr noalias %src16,
+; PRESSURE-LABEL: 'uniform_narrow_widening_add'
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 6 registers
+; PRESSURE: LV(REG): Calculating conservative max register usage:
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 10 registers
+;
+ ptr noalias %wide,
+ ptr noalias %out16,
+ ptr noalias %out, i16 %narrow,
+ i64 %n) {
+entry:
+ %narrow.ext = sext i16 %narrow to i32
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %src16.gep = getelementptr i16, ptr %src16, i64 %iv
+ %narrow.value = load i16, ptr %src16.gep, align 2
+ %wide.gep = getelementptr i32, ptr %wide, i64 %iv
+ %a = load i32, ptr %wide.gep, align 4
+ %r = add i32 %a, %narrow.ext
+ %out16.gep = getelementptr i16, ptr %out16, i64 %iv
+ %out.gep = getelementptr i32, ptr %out, i64 %iv
+ store i16 %narrow.value, ptr %out16.gep, align 2
+ store i32 %r, ptr %out.gep, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @non_widening_sub_operand_order(ptr noalias %src16,
+; PRESSURE-LABEL: 'non_widening_sub_operand_order'
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 6 registers
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 12 registers
+; PRESSURE: LV(REG): Calculating conservative max register usage:
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 6 registers
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 12 registers
+;
+ ptr noalias %wide,
+ ptr noalias %out0,
+ ptr noalias %out1,
+ ptr noalias %out2, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %src.gep = getelementptr i16, ptr %src16, i64 %iv
+ %x = load i16, ptr %src.gep, align 2
+ %x.ext = sext i16 %x to i32
+ %wide.gep = getelementptr i32, ptr %wide, i64 %iv
+ %a = load i32, ptr %wide.gep, align 4
+ %r = sub i32 %x.ext, %a
+ %out0.gep = getelementptr i32, ptr %out0, i64 %iv
+ %out1.gep = getelementptr i32, ptr %out1, i64 %iv
+ %out2.gep = getelementptr i32, ptr %out2, i64 %iv
+ store i32 %r, ptr %out0.gep, align 4
+ store i32 %a, ptr %out1.gep, align 4
+ store i32 %x.ext, ptr %out2.gep, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-pressure.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-pressure.ll
new file mode 100644
index 0000000000000..f03525afe0bbf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-pressure.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: (Found feasible scalable VF|Selecting VF)" --filter "LV\(REG\): (Calculating conservative max register usage|VF = vscale x (8|16)$|RegisterClass: RISCV::VRRC, (12|18|24|34) registers|Cost of [0-9]+ from 1 spills of RISCV::VRRC|Not considering vector loop)" --filter "Cost for VF vscale x 4:" --version 6
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -vectorizer-maximize-bandwidth -force-vector-interleave=1 -force-target-num-vector-regs=64 -debug-only=loop-vectorize,vplan -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PRESSURE
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -vectorizer-maximize-bandwidth -force-vector-interleave=1 -force-target-num-vector-regs=5 -debug-only=loop-vectorize,vplan -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SOFT-HARD --implicit-check-not='LV(REG): Not considering vector loop of width vscale x 4' --implicit-check-not='Cost for VF vscale x 8:'
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -vectorizer-maximize-bandwidth -force-vector-interleave=1 -force-target-num-vector-regs=18 -debug-only=loop-vectorize,vplan -disable-output < %s 2>&1 | FileCheck %s --check-prefix=HARD-BOUNDARY --implicit-check-not='LV(REG): Not considering vector loop of width vscale x 8' --implicit-check-not='Cost for VF vscale x 16:'
+
+define i32 @predicated_three_way_reduction(ptr %a, ptr %b, ptr %cond.a,
+; PRESSURE-LABEL: 'predicated_three_way_reduction'
+; PRESSURE: LV: Found feasible scalable VF = vscale x 16
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 12 registers
+; PRESSURE: LV(REG): VF = vscale x 16
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
+; PRESSURE: LV(REG): Calculating conservative max register usage:
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 18 registers
+; PRESSURE: LV(REG): VF = vscale x 16
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 34 registers
+; PRESSURE: Cost for VF vscale x 4: 26 (Estimated cost per lane: 3.25)
+; PRESSURE: LV: Selecting VF: vscale x 16.
+;
+; SOFT-HARD-LABEL: 'predicated_three_way_reduction'
+; SOFT-HARD: LV: Found feasible scalable VF = vscale x 16
+; SOFT-HARD: LV(REG): VF = vscale x 8
+; SOFT-HARD: LV(REG): RegisterClass: RISCV::VRRC, 12 registers
+; SOFT-HARD: LV(REG): VF = vscale x 16
+; SOFT-HARD: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
+; SOFT-HARD: LV(REG): Calculating conservative max register usage:
+; SOFT-HARD: LV(REG): VF = vscale x 8
+; SOFT-HARD: LV(REG): RegisterClass: RISCV::VRRC, 18 registers
+; SOFT-HARD: LV(REG): VF = vscale x 16
+; SOFT-HARD: LV(REG): RegisterClass: RISCV::VRRC, 34 registers
+; SOFT-HARD: LV(REG): Cost of 2 from 1 spills of RISCV::VRRC
+; SOFT-HARD: Cost for VF vscale x 4: 30 (Estimated cost per lane: 3.75)
+; SOFT-HARD: LV(REG): Not considering vector loop of width vscale x 8 because it uses too many registers
+; SOFT-HARD: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; SOFT-HARD: LV: Selecting VF: vscale x 4.
+;
+; HARD-BOUNDARY-LABEL: 'predicated_three_way_reduction'
+; HARD-BOUNDARY: LV: Found feasible scalable VF = vscale x 16
+; HARD-BOUNDARY: LV(REG): VF = vscale x 8
+; HARD-BOUNDARY: LV(REG): RegisterClass: RISCV::VRRC, 12 registers
+; HARD-BOUNDARY: LV(REG): VF = vscale x 16
+; HARD-BOUNDARY: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
+; HARD-BOUNDARY: LV(REG): Calculating conservative max register usage:
+; HARD-BOUNDARY: LV(REG): VF = vscale x 8
+; HARD-BOUNDARY: LV(REG): RegisterClass: RISCV::VRRC, 18 registers
+; HARD-BOUNDARY: LV(REG): VF = vscale x 16
+; HARD-BOUNDARY: LV(REG): RegisterClass: RISCV::VRRC, 34 registers
+; HARD-BOUNDARY: Cost for VF vscale x 4: 26 (Estimated cost per lane: 3.25)
+; HARD-BOUNDARY: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; HARD-BOUNDARY: LV: Selecting VF: vscale x 8.
+;
+ ptr %cond.b, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.next, %latch ]
+ %cond.a.gep = getelementptr inbounds nuw i8, ptr %cond.a, i64 %iv
+ %cond.a.val = load i8, ptr %cond.a.gep, align 1
+ %take.a = icmp ne i8 %cond.a.val, 0
+ br i1 %take.a, label %then.a, label %check.b
+
+then.a:
+ %a.gep = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %a.val = load i8, ptr %a.gep, align 1
+ %a.ext = zext i8 %a.val to i32
+ %sum.a = add nsw i32 %sum, %a.ext
+ br label %latch
+
+check.b:
+ %cond.b.gep = getelementptr inbounds nuw i8, ptr %cond.b, i64 %iv
+ %cond.b.val = load i8, ptr %cond.b.gep, align 1
+ %take.b = icmp ne i8 %cond.b.val, 0
+ br i1 %take.b, label %then.b, label %else.b
+
+then.b:
+ %b.gep = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %b.val = load i8, ptr %b.gep, align 1
+ %b.ext = zext i8 %b.val to i32
+ %sum.b = add nsw i32 %sum, %b.ext
+ br label %latch
+
+else.b:
+ br label %latch
+
+latch:
+ %sum.next = phi i32 [ %sum.a, %then.a ], [ %sum.b, %then.b ],
+ [ %sum, %else.b ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %sum.next
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-same-value.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-same-value.ll
new file mode 100644
index 0000000000000..c8962e902f002
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-widening-same-value.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Selecting VF" --filter "LV\(REG\): (Calculating conservative max register usage|VF = (vscale x 8|8|16)$|RegisterClass: RISCV::VRRC, (2|4|8) registers|Not considering vector loop)" --version 6
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -vectorizer-maximize-bandwidth -force-vector-interleave=1 -force-target-num-vector-regs=64 -debug-only=loop-vectorize,vplan -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PRESSURE
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -scalable-vectorization=off -vectorizer-maximize-bandwidth -force-vector-interleave=1 -force-target-num-vector-regs=4 -debug-only=loop-vectorize,vplan -disable-output < %s 2>&1 | FileCheck %s --check-prefix=FIXED
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -force-vector-width=16 -force-vector-interleave=1 -debug-only=loop-vectorize,vplan -disable-output < %s 2>&1 | FileCheck %s --check-prefix=FORCED-FIXED --implicit-check-not='LV(REG): Calculating conservative max register usage:' --implicit-check-not='LV(REG): Not considering vector loop'
+
+define void @same_value_two_edges(ptr %src, ptr %dst, i64 %n) {
+; PRESSURE-LABEL: 'same_value_two_edges'
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 2 registers
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 4 registers
+; PRESSURE: LV(REG): Calculating conservative max register usage:
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 2 registers
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 2 registers
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 4 registers
+; PRESSURE: LV(REG): VF = vscale x 8
+; PRESSURE: LV(REG): RegisterClass: RISCV::VRRC, 8 registers
+; PRESSURE: LV: Selecting VF: vscale x 8.
+;
+; FIXED-LABEL: 'same_value_two_edges'
+; FIXED: LV(REG): VF = 8
+; FIXED: LV(REG): RegisterClass: RISCV::VRRC, 2 registers
+; FIXED: LV(REG): VF = 16
+; FIXED: LV(REG): RegisterClass: RISCV::VRRC, 4 registers
+; FIXED: LV(REG): Calculating conservative max register usage:
+; FIXED: LV(REG): RegisterClass: RISCV::VRRC, 2 registers
+; FIXED: LV(REG): RegisterClass: RISCV::VRRC, 2 registers
+; FIXED: LV(REG): VF = 8
+; FIXED: LV(REG): RegisterClass: RISCV::VRRC, 4 registers
+; FIXED: LV(REG): VF = 16
+; FIXED: LV(REG): RegisterClass: RISCV::VRRC, 8 registers
+; FIXED: LV(REG): Not considering vector loop of width 16 because it uses too many registers
+; FIXED: LV: Selecting VF: 8.
+; FIXED: LV(REG): VF = 8
+; FIXED: LV(REG): RegisterClass: RISCV::VRRC, 2 registers
+;
+; FORCED-FIXED-LABEL: 'same_value_two_edges'
+; FORCED-FIXED: LV(REG): VF = 16
+; FORCED-FIXED: LV(REG): RegisterClass: RISCV::VRRC, 4 registers
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
+ %gep = getelementptr i16, ptr %src, i64 %iv
+ %x = load i16, ptr %gep, align 2
+ %ext = sext i16 %x to i32
+ %twice = add i32 %ext, %ext
+ %out = getelementptr i32, ptr %dst, i64 %iv
+ store i32 %twice, ptr %out, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list