[llvm] [RISCV] Account for VL toggle cost when splitting VP memory intrinsics (PR #223992)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 17 05:47:18 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/223992
>From f20c6eebb16f1afd01286ccda634f19c8de46c22 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 16 Sep 2026 20:14:20 +0800
Subject: [PATCH] [RISCV] Account for VL toggle cost when splitting VP memory
intrinsics
Not an alternative to #222836 since this doesn't handle all cases, but something we should do anyway in the TTI to be accurate.
For vp.load/vp.gather/vp.strided.load and their store counterparts, when they're split they introduce extra vl toggles which involve computation on the evl, e.g.
vsetvli a2, zero, e8, mf4, ta, ma
vmv1r.v v8, v0
csrr a2, vlenb
srli a3, a2, 3
vslidedown.vx v0, v0, a3
sub a3, a1, a2
sltu a4, a1, a3
addi a4, a4, -1
slli a5, a2, 3
and a3, a4, a3
add a5, a0, a5
vsetvli zero, a3, e64, m8, ta, ma
vle64.v v16, (a5), v0.t
bltu a1, a2, .LBB44_2
# %bb.1:
mv a1, a2
.LBB44_2:
vmv1r.v v0, v8
vsetvli zero, a1, e64, m8, ta, ma
vle64.v v8, (a0), v0.t
Account for some overhead in the cost to influence the loop vectorizer to avoid picking VFs that lead to types that need split.
---
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 33 ++++++++++++++-
.../Analysis/CostModel/RISCV/vp-intrinsics.ll | 10 ++---
.../RISCV/strided-access-wide-stride.ll | 42 +++++++++----------
.../RISCV/strided-accesses-narrow-iv.ll | 26 ++++++------
4 files changed, 70 insertions(+), 41 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 6877d54384a52..43b4791998d39 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1160,6 +1160,15 @@ RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
CostKind != TTI::TCK_RecipThroughput)
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+ // Splitting involves additional evl arithmetic and vl toggles.
+ InstructionCost SplitCost = 0;
+ if (MICA.getID() == Intrinsic::vp_load ||
+ MICA.getID() == Intrinsic::vp_store) {
+ auto LT = getTypeLegalizationCost(Src);
+ if (LT.first > 1)
+ SplitCost += LT.first * TTI::TCC_Expensive;
+ }
+
return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
}
@@ -1300,6 +1309,8 @@ RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
MICA.getID() == Intrinsic::vp_gather;
unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
Type *DataTy = MICA.getDataType();
+ Type *PtrTy = DataTy->getWithNewType(
+ DL.getAddressType(DataTy->getContext(), MICA.getAddressSpace()));
Align Alignment = MICA.getAlignment();
if (CostKind != TTI::TCK_RecipThroughput)
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
@@ -1310,12 +1321,24 @@ RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
!isLegalMaskedScatter(DataTy, Align(Alignment))))
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+ // Splitting vp intrinsics involves additional evl arithmetic and vl toggles.
+ InstructionCost SplitCost = 0;
+ if (MICA.getID() == Intrinsic::vp_gather ||
+ MICA.getID() == Intrinsic::vp_scatter) {
+ auto DataLT = getTypeLegalizationCost(DataTy);
+ auto PtrLT = getTypeLegalizationCost(PtrTy);
+ if (DataLT.first > 1)
+ SplitCost += DataLT.first * TTI::TCC_Expensive;
+ if (PtrLT.first > 1)
+ SplitCost += PtrLT.first * TTI::TCC_Expensive;
+ }
+
// Cost is proportional to the number of memory operations implied. For
// scalable vectors, we use an estimate on that number since we don't
// know exactly what VL will be.
auto &VTy = *cast<VectorType>(DataTy);
unsigned NumLoads = getEstimatedVLFor(&VTy);
- return NumLoads * TTI::TCC_Basic;
+ return SplitCost + NumLoads * TTI::TCC_Basic;
}
InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
@@ -1373,12 +1396,18 @@ RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
if (CostKind == TTI::TCK_CodeSize)
return TTI::TCC_Basic;
+ // Splitting vp intrinsics involves additional evl arithmetic and vl toggles.
+ InstructionCost SplitCost = 0;
+ auto LT = getTypeLegalizationCost(DataTy);
+ if (LT.first > 1)
+ SplitCost += LT.first * TTI::TCC_Expensive;
+
// Cost is proportional to the number of memory operations implied. For
// scalable vectors, we use an estimate on that number since we don't
// know exactly what VL will be.
auto &VTy = *cast<VectorType>(DataTy);
unsigned NumLoads = getEstimatedVLFor(&VTy);
- return NumLoads * TTI::TCC_Basic;
+ return SplitCost + NumLoads * TTI::TCC_Basic;
}
InstructionCost
diff --git a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
index df764653b5085..0f95ad61ec2f6 100644
--- a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
@@ -1009,7 +1009,7 @@ define void @gather() {
; ARGBASED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %9 = call <vscale x 2 x i8> @llvm.vp.gather.nxv2i8.nxv2p0(<vscale x 2 x ptr> poison, <vscale x 2 x i1> poison, i32 poison)
; ARGBASED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %10 = call <vscale x 4 x i8> @llvm.vp.gather.nxv4i8.nxv4p0(<vscale x 4 x ptr> poison, <vscale x 4 x i1> poison, i32 poison)
; ARGBASED-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %11 = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0(<vscale x 8 x ptr> poison, <vscale x 8 x i1> poison, i32 poison)
-; ARGBASED-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %12 = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> poison, <vscale x 16 x i1> poison, i32 poison)
+; ARGBASED-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %12 = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> poison, <vscale x 16 x i1> poison, i32 poison)
; ARGBASED-NEXT: Cost Model: Invalid cost for instruction: %13 = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> poison, <vscale x 2 x i1> poison, i32 poison)
; ARGBASED-NEXT: Cost Model: Invalid cost for instruction: %14 = call <vscale x 4 x i64> @llvm.vp.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> poison, <vscale x 4 x i1> poison, i32 poison)
; ARGBASED-NEXT: Cost Model: Invalid cost for instruction: %15 = call <vscale x 8 x i64> @llvm.vp.gather.nxv8i64.nxv8p0(<vscale x 8 x ptr> poison, <vscale x 8 x i1> poison, i32 poison)
@@ -1067,7 +1067,7 @@ define void @scatter() {
; ARGBASED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> poison, <vscale x 2 x ptr> poison, <vscale x 2 x i1> poison, i32 poison)
; ARGBASED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.vp.scatter.nxv4i8.nxv4p0(<vscale x 4 x i8> poison, <vscale x 4 x ptr> poison, <vscale x 4 x i1> poison, i32 poison)
; ARGBASED-NEXT: Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> poison, <vscale x 8 x ptr> poison, <vscale x 8 x i1> poison, i32 poison)
-; ARGBASED-NEXT: Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> poison, <vscale x 16 x ptr> poison, <vscale x 16 x i1> poison, i32 poison)
+; ARGBASED-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> poison, <vscale x 16 x ptr> poison, <vscale x 16 x i1> poison, i32 poison)
; ARGBASED-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> poison, <vscale x 2 x ptr> poison, <vscale x 2 x i1> poison, i32 poison)
; ARGBASED-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.vp.scatter.nxv4i64.nxv4p0(<vscale x 4 x i64> poison, <vscale x 4 x ptr> poison, <vscale x 4 x i1> poison, i32 poison)
; ARGBASED-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.vp.scatter.nxv8i64.nxv8p0(<vscale x 8 x i64> poison, <vscale x 8 x ptr> poison, <vscale x 8 x i1> poison, i32 poison)
@@ -1137,7 +1137,7 @@ define void @strided_load() {
; ARGBASED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %t25 = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
; ARGBASED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %t27 = call <vscale x 4 x i64> @llvm.experimental.vp.strided.load.nxv4i64.p0.i64(ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
; ARGBASED-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x i64> @llvm.experimental.vp.strided.load.nxv8i64.p0.i64(ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
-; ARGBASED-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x i64> @llvm.experimental.vp.strided.load.nxv16i64.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
+; ARGBASED-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %t31 = call <vscale x 16 x i64> @llvm.experimental.vp.strided.load.nxv16i64.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
; ARGBASED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; TYPEBASED-LABEL: 'strided_load'
@@ -1164,7 +1164,7 @@ define void @strided_load() {
; TYPEBASED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %t25 = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
; TYPEBASED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %t27 = call <vscale x 4 x i64> @llvm.experimental.vp.strided.load.nxv4i64.p0.i64(ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
; TYPEBASED-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %t29 = call <vscale x 8 x i64> @llvm.experimental.vp.strided.load.nxv8i64.p0.i64(ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
-; TYPEBASED-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %t31 = call <vscale x 16 x i64> @llvm.experimental.vp.strided.load.nxv16i64.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
+; TYPEBASED-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %t31 = call <vscale x 16 x i64> @llvm.experimental.vp.strided.load.nxv16i64.p0.i64(ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
; TYPEBASED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
%ti1_2 = call <2 x i1> @llvm.experimental.vp.strided.load.v2i1.i64(ptr undef, i64 undef, <2 x i1> undef, i32 undef)
@@ -1215,7 +1215,7 @@ define void @strided_store() {
; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.experimental.vp.strided.store.nxv2i64.p0.i64(<vscale x 2 x i64> undef, ptr undef, i64 undef, <vscale x 2 x i1> undef, i32 undef)
; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vp.strided.store.nxv4i64.p0.i64(<vscale x 4 x i64> undef, ptr undef, i64 undef, <vscale x 4 x i1> undef, i32 undef)
; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vp.strided.store.nxv8i64.p0.i64(<vscale x 8 x i64> undef, ptr undef, i64 undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT: Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vp.strided.store.nxv16i64.p0.i64(<vscale x 16 x i64> undef, ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.experimental.vp.strided.store.nxv16i64.p0.i64(<vscale x 16 x i64> undef, ptr undef, i64 undef, <vscale x 16 x i1> undef, i32 undef)
; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
call void @llvm.experimental.vp.strided.store.v2i8.i64(<2 x i8> undef, ptr undef, i64 undef, <2 x i1> undef, i32 undef)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
index 0b9617479b0de..8ed29c2306413 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-access-wide-stride.ll
@@ -15,27 +15,27 @@ define void @stride_sext_exceeds_i32_max(ptr noalias readonly %src, ptr noalias
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 16 x ptr> poison, <vscale x 16 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i16> @llvm.stepvector.nxv16i16()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i16> poison, i16 [[START]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i16> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i16> poison, <vscale x 16 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 16 x i16> [[BROADCAST_SPLAT2]], [[TMP3]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.stepvector.nxv8i16()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[START]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 8 x i16> [[BROADCAST_SPLAT2]], [[TMP3]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i16> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i16> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i16
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 16 x i16> poison, i16 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 16 x i16> [[BROADCAST_SPLATINSERT3]], <vscale x 16 x i16> poison, <vscale x 16 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i16> [[VEC_IND]] to <vscale x 16 x i64>
-; CHECK-NEXT: [[TMP7:%.*]] = mul nsw <vscale x 16 x i64> [[TMP6]], splat (i64 3000000000)
-; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[SRC]], <vscale x 16 x i64> [[TMP7]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[WIDE_GEP]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP4]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP4]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT3]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 8 x i16> [[VEC_IND]] to <vscale x 8 x i64>
+; CHECK-NEXT: [[TMP7:%.*]] = mul nsw <vscale x 8 x i64> [[TMP6]], splat (i64 3000000000)
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[SRC]], <vscale x 8 x i64> [[TMP7]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0(<vscale x 8 x ptr> align 1 [[WIDE_GEP]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP4]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[WIDE_MASKED_GATHER]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP4]])
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP4]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i16> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i16> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -73,18 +73,18 @@ define void @stride_zext_exceeds_i32_max(ptr noalias readonly %src, ptr noalias
; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[START]] to i64
; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[TMP3]], 3000000000
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 16 x ptr> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
+; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[INDEX]] to i64
; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 3000000000
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP5]], i64 [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.p0.i64(ptr align 1 [[TMP9]], i64 3000000000, <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP10]], <vscale x 16 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
+; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP9]], i64 3000000000, <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP10]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP6]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll
index 1a1d3a4d16d42..facc909ad0835 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses-narrow-iv.ll
@@ -285,25 +285,25 @@ define void @narrow_iv_i8_sext_i64_wrapping(ptr noalias %arr, ptr noalias %out)
; RV64-NEXT: [[ENTRY:.*:]]
; RV64-NEXT: br label %[[LOOP:.*]]
; RV64: [[LOOP]]:
-; RV64-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x ptr> poison, ptr [[OUT]], i64 0
-; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 16 x ptr> poison, <vscale x 16 x i32> zeroinitializer
-; RV64-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.stepvector.nxv16i8()
-; RV64-NEXT: [[TMP1:%.*]] = mul <vscale x 16 x i8> [[TMP0]], splat (i8 4)
+; RV64-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[OUT]], i64 0
+; RV64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
+; RV64-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
+; RV64-NEXT: [[TMP1:%.*]] = mul <vscale x 8 x i8> [[TMP0]], splat (i8 4)
; RV64-NEXT: br label %[[EXIT:.*]]
; RV64: [[EXIT]]:
-; RV64-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i8> [ [[TMP1]], %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[EXIT]] ]
+; RV64-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i8> [ [[TMP1]], %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[EXIT]] ]
; RV64-NEXT: [[AVL:%.*]] = phi i32 [ 64, %[[LOOP]] ], [ [[AVL_NEXT:%.*]], %[[EXIT]] ]
-; RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
+; RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)
; RV64-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
; RV64-NEXT: [[TMP4:%.*]] = shl i8 [[TMP3]], 2
-; RV64-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP4]], i64 0
-; RV64-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-; RV64-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[VEC_IND]] to <vscale x 16 x i64>
-; RV64-NEXT: [[WIDE_GEP:%.*]] = getelementptr [1024 x i8], ptr [[ARR]], <vscale x 16 x i64> [[TMP5]]
-; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[WIDE_GEP]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]])
-; RV64-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]])
+; RV64-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[TMP4]], i64 0
+; RV64-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
+; RV64-NEXT: [[TMP5:%.*]] = sext <vscale x 8 x i8> [[VEC_IND]] to <vscale x 8 x i64>
+; RV64-NEXT: [[WIDE_GEP:%.*]] = getelementptr [1024 x i8], ptr [[ARR]], <vscale x 8 x i64> [[TMP5]]
+; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0(<vscale x 8 x ptr> align 1 [[WIDE_GEP]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP2]])
+; RV64-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[WIDE_MASKED_GATHER]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP2]])
; RV64-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
-; RV64-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; RV64-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i8> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; RV64-NEXT: [[TMP6:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; RV64-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]], !llvm.loop [[LOOP6:![0-9]+]]
; RV64: [[MIDDLE_BLOCK]]:
More information about the llvm-commits
mailing list