[llvm] [SLP] Enable widening strided revectorization of vector stores (PR #198920)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Sun May 24 18:44:51 PDT 2026
https://github.com/bababuck updated https://github.com/llvm/llvm-project/pull/198920
>From fb57851296cdffce18497cc650a471a82018b0ff Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 10 Apr 2026 12:27:50 -0700
Subject: [PATCH 1/3] [SLP] Enable revectorization of strided stores
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 20 +++++++++++++------
.../RISCV/revec-strided-store.ll | 13 ++++--------
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index ffa8f2ed32b22..1836a782b21e1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17432,11 +17432,15 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
// We know that we can merge the stores. Calculate the cost.
InstructionCost VecStCost;
if (E->State == TreeEntry::StridedVectorize) {
+ const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
+ FixedVectorType *StridedStoreTy = SPtrInfo.Ty;
+ assert(StridedStoreTy && "Missing StridedPointerInfo for tree entry.");
Align CommonAlignment =
computeCommonAlignment<StoreInst>(UniqueValues.getArrayRef());
VecStCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_store,
- VecTy, BaseSI->getPointerOperand(),
+ StridedStoreTy,
+ BaseSI->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment),
CostKind);
} else {
@@ -23270,6 +23274,10 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
Type *StrideTy = DL->getIndexType(SI->getPointerOperandType());
const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
+ FixedVectorType *StridedStoreTy = SPtrInfo.Ty;
+ assert(StridedStoreTy && "Missing StridedPointerInfo for tree entry.");
+ unsigned StridedStoreEC =
+ StridedStoreTy->getElementCount().getKnownMinValue();
Value *Stride = SPtrInfo.StrideVal;
assert(Stride && "Missing StridedPointerInfo for tree entry.");
Value *StrideVal =
@@ -23279,12 +23287,14 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
StrideVal,
ConstantInt::getSigned(
StrideTy, static_cast<int>(DL->getTypeAllocSize(ScalarTy))));
+ if (StridedStoreTy != VecTy)
+ VecValue = Builder.CreateBitOrPointerCast(VecValue, StridedStoreTy);
auto *Inst = Builder.CreateIntrinsic(
Intrinsic::experimental_vp_strided_store,
- {VecTy, Ptr->getType(), StrideTy},
+ {StridedStoreTy, Ptr->getType(), StrideTy},
{VecValue, Ptr, StrideVal,
- Builder.getAllOnesMask(VecTy->getElementCount()),
- Builder.getInt32(E->Scalars.size())});
+ Builder.getAllOnesMask(ElementCount::getFixed(StridedStoreEC)),
+ Builder.getInt32(StridedStoreEC)});
Inst->addParamAttr(
/*ArgNo=*/1,
Attribute::getWithAlignment(Inst->getContext(), CommonAlignment));
@@ -26989,8 +26999,6 @@ bool StoreChainContext::initializeContext(
BoUpSLP &R, const DataLayout &DL, const TargetTransformInfo &TTI,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited) {
- assert((Stride == 1 || !SLPReVec) &&
- "Strided stores not supported for revectorization");
if (!Visited
.insert({Operands.front(),
cast<StoreInst>(Operands.front())->getValueOperand(),
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-store.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-store.ll
index 47b806acb437a..9d75c89949e0d 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-store.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-store.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple=riscv64 -mcpu=sifive-p870 -passes=slp-vectorizer -S -slp-revec -slp-threshold=-100 %s | FileCheck %s
+; RUN: opt -mtriple=riscv64 -mcpu=sifive-p870 -passes=slp-vectorizer -S -slp-revec -slp-enable-strided-stores -slp-threshold=-100 %s | FileCheck %s
; Strided load into strided store
define void @strided_load_and_store(ptr %in, ptr %out) {
@@ -7,11 +7,8 @@ define void @strided_load_and_store(ptr %in, ptr %out) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr align 2 [[IN:%.*]], i64 16, <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i64> [[TMP0]] to <16 x i8>
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[OUT:%.*]], i64 16
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: store <8 x i8> [[TMP1]], ptr [[OUT]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: store <8 x i8> [[TMP2]], ptr [[TMP3]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[TMP4]] to <2 x i64>
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v2i64.p0.i64(<2 x i64> [[TMP2]], ptr align 2 [[OUT:%.*]], i64 16, <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: ret void
;
entry:
@@ -27,9 +24,7 @@ entry:
; Base case of strided store, implicitly is widened
define void @widened_strided_store(ptr %out0) {
; CHECK-LABEL: @widened_strided_store(
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr null, i64 16
-; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr [[TMP1]], align 2
-; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr null, align 2
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v2i64.p0.i64(<2 x i64> zeroinitializer, ptr align 2 null, i64 32, <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: ret void
;
%1 = getelementptr i16, ptr null, i64 16
>From 976d1e1e4940615c2c81e16098cc92ac57c29cde Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 21 May 2026 23:11:13 -0700
Subject: [PATCH 2/3] [SLP] Cost the bitcast
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1836a782b21e1..72404bef5c9d7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17443,6 +17443,11 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
BaseSI->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment),
CostKind);
+ if (StridedStoreTy != VecTy)
+ VecStCost +=
+ TTI->getCastInstrCost(Instruction::BitCast, VecTy, StridedStoreTy,
+ getCastContextHint(*E), CostKind);
+
} else {
assert(E->State == TreeEntry::Vectorize &&
"Expected either strided or consecutive stores.");
>From 3c24ddf3f277a7ed9a11b4ed2b20ff23f4c86df6 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 21 May 2026 23:14:25 -0700
Subject: [PATCH 3/3] [SLP] Use getNumElements
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 72404bef5c9d7..26a20d47a01b7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23281,8 +23281,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
FixedVectorType *StridedStoreTy = SPtrInfo.Ty;
assert(StridedStoreTy && "Missing StridedPointerInfo for tree entry.");
- unsigned StridedStoreEC =
- StridedStoreTy->getElementCount().getKnownMinValue();
+ unsigned StridedStoreEC = getNumElements(StridedStoreTy);
Value *Stride = SPtrInfo.StrideVal;
assert(Stride && "Missing StridedPointerInfo for tree entry.");
Value *StrideVal =
More information about the llvm-commits
mailing list