[llvm] [SLP] Pass along stride information when costing strided loads and stores (PR #227823)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 11:53:00 PDT 2026
https://github.com/bababuck created https://github.com/llvm/llvm-project/pull/227823
In cases of constant strides, can give a more accurate cost when accounting for a known stride.
Currently, only RISCV backend makes use of this information.
Assisted By: Codex
>From 53cf5131ca0581c249f912bd1ae0a55e1490e8c3 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Tue, 29 Sep 2026 13:42:59 -0700
Subject: [PATCH] [SLP] Pass along stride information when costing strided
loads and stores
In cases of constant strides, can give a more accurate cost when accounting
for a known stride.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 45 ++++++++++++-------
.../SLPVectorizer/SLPMemoryUtils.cpp | 12 +++++
.../Vectorize/SLPVectorizer/SLPMemoryUtils.h | 5 +++
.../RISCV/basic-strided-stores.ll | 45 +++++++------------
.../SLPVectorizer/RISCV/complex-loads.ll | 12 +++--
5 files changed, 68 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 30de440e44721a..04f4dd912cd509 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14610,18 +14610,20 @@ void BoUpSLP::transformNodes() {
BaseLI->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(BaseLI->getPointerOperand())) +
getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, CostKind, Mask);
+ Type *StrideTy = DL->getIndexType(
+ cast<LoadInst>(E.Scalars.front())->getPointerOperand()->getType());
InstructionCost StridedCost = TTI->getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_load,
- VecTy, BaseLI->getPointerOperand(),
- /*VariableMask=*/false, CommonAlignment,
- BaseLI),
+ MemIntrinsicCostAttributes(
+ Intrinsic::experimental_vp_strided_load, VecTy,
+ BaseLI->getPointerOperand(),
+ /*VariableMask=*/false, CommonAlignment, BaseLI,
+ ConstantInt::getSigned(
+ StrideTy,
+ -static_cast<int64_t>(DL->getTypeAllocSize(ScalarTy)))),
CostKind);
if (StridedCost < OriginalVecCost || ForceStridedLoads) {
// Strided load is more profitable than consecutive load + reverse -
// transform the node to strided load.
- Type *StrideTy = DL->getIndexType(cast<LoadInst>(E.Scalars.front())
- ->getPointerOperand()
- ->getType());
StridedPtrInfo SPtrInfo;
SPtrInfo.StrideVal = ConstantInt::get(StrideTy, 1);
SPtrInfo.Ty = VecTy;
@@ -14651,19 +14653,21 @@ void BoUpSLP::transformNodes() {
BaseSI->getPointerAddressSpace(), CostKind,
getOperandInfo(E.getOperand(0))) +
getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, CostKind, Mask);
+ Type *StrideTy = DL->getIndexType(
+ cast<StoreInst>(E.Scalars.front())->getPointerOperand()->getType());
InstructionCost StridedCost = TTI->getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_store,
- VecTy, BaseSI->getPointerOperand(),
- /*VariableMask=*/false, CommonAlignment,
- BaseSI),
+ MemIntrinsicCostAttributes(
+ Intrinsic::experimental_vp_strided_store, VecTy,
+ BaseSI->getPointerOperand(),
+ /*VariableMask=*/false, CommonAlignment, BaseSI,
+ ConstantInt::getSigned(
+ StrideTy,
+ -static_cast<int64_t>(DL->getTypeAllocSize(ScalarTy)))),
CostKind);
if (StridedCost < OriginalVecCost) {
// Strided store is more profitable than reverse + consecutive store -
// transform the node to strided store.
E.State = TreeEntry::StridedVectorize;
- Type *StrideTy = DL->getIndexType(cast<StoreInst>(E.Scalars.front())
- ->getPointerOperand()
- ->getType());
StridedPtrInfo SPtrInfo;
SPtrInfo.StrideVal = ConstantInt::getSigned(StrideTy, -1);
SPtrInfo.Ty = VecTy;
@@ -17479,10 +17483,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
assert(StridedLoadTy && "Missing StridedPointerInfo for tree entry.");
Align CommonAlignment =
computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
+ Type *StrideTy = DL->getIndexType(LI0->getPointerOperand()->getType());
+ Value *Stride = getStrideBytesIfConstant(SPtrInfo.StrideVal, ScalarTy,
+ StrideTy, *DL);
VecLdCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_load,
StridedLoadTy, LI0->getPointerOperand(),
- /*VariableMask=*/false, CommonAlignment),
+ /*VariableMask=*/false, CommonAlignment,
+ /*I=*/nullptr, Stride),
CostKind);
if (StridedLoadTy != VecTy)
VecLdCost +=
@@ -17596,11 +17604,16 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
assert(StridedStoreTy && "Missing StridedPointerInfo for tree entry.");
Align CommonAlignment =
computeCommonAlignment<StoreInst>(UniqueValues.getArrayRef());
+ Type *StrideTy =
+ DL->getIndexType(BaseSI->getPointerOperand()->getType());
+ Value *Stride = getStrideBytesIfConstant(SPtrInfo.StrideVal, ScalarTy,
+ StrideTy, *DL);
VecStCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_store,
StridedStoreTy,
BaseSI->getPointerOperand(),
- /*VariableMask=*/false, CommonAlignment),
+ /*VariableMask=*/false, CommonAlignment,
+ /*I=*/nullptr, Stride),
CostKind);
if (StridedStoreTy != VecTy)
VecStCost +=
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
index 8279c277a1f047..7307f0956d5d86 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
@@ -40,6 +40,18 @@ using namespace llvm;
namespace llvm::slpvectorizer {
+ConstantInt *getStrideBytesIfConstant(Value *Stride, Type *ScalarTy,
+ Type *StrideTy, const DataLayout &DL) {
+ auto *CI = dyn_cast_or_null<ConstantInt>(Stride);
+ if (!CI)
+ return nullptr;
+
+ unsigned StrideWidth = cast<IntegerType>(StrideTy)->getBitWidth();
+ APInt ElementSize(StrideWidth, DL.getTypeAllocSize(ScalarTy));
+ APInt StrideValue = CI->getValue().sextOrTrunc(StrideWidth) * ElementSize;
+ return ConstantInt::get(StrideTy->getContext(), StrideValue);
+}
+
bool arePointersCompatible(Value *Ptr1, Value *Ptr2,
const TargetLibraryInfo &TLI, unsigned MaxDepth,
bool CompareOpcodes) {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
index d1df27e1fb6c3d..afb84e8150f64d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
@@ -22,6 +22,7 @@
namespace llvm {
class AssumptionCache;
class BasicBlock;
+class ConstantInt;
class DataLayout;
class DominatorTree;
class FixedVectorType;
@@ -40,6 +41,10 @@ bool arePointersCompatible(Value *Ptr1, Value *Ptr2,
const TargetLibraryInfo &TLI, unsigned MaxDepth,
bool CompareOpcodes = true);
+/// Returns the byte stride if \p Stride is a constant, or nullptr otherwise.
+ConstantInt *getStrideBytesIfConstant(Value *Stride, Type *ScalarTy,
+ Type *StrideTy, const DataLayout &DL);
+
/// Calculates minimal alignment as a common alignment.
template <typename T> Align computeCommonAlignment(ArrayRef<Value *> VL);
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-stores.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-stores.ll
index a738dd27bcaf77..192669794e11f8 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-stores.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-stores.ll
@@ -528,45 +528,34 @@ define void @vf_ordering_issue(ptr %pl, ptr %ps) {
; CHECK-LABEL: define void @vf_ordering_issue(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 38
; CHECK-NEXT: [[GEP_L14:%.*]] = getelementptr i8, ptr [[PL]], i64 92
-; CHECK-NEXT: [[GEP_S11:%.*]] = getelementptr i8, ptr [[PL]], i64 33
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L20:%.*]] = getelementptr i8, ptr [[PL]], i64 13
-; CHECK-NEXT: [[GEP_L21:%.*]] = getelementptr i8, ptr [[PL]], i64 83
-; CHECK-NEXT: [[GEP_L22:%.*]] = getelementptr i8, ptr [[PL]], i64 32
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x ptr> poison, ptr [[PL]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x ptr> [[TMP1]], <8 x ptr> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, <8 x ptr> [[TMP2]], <8 x i64> <i64 33, i64 4, i64 6, i64 8, i64 10, i64 13, i64 83, i64 32>
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 15
; CHECK-NEXT: [[GEP_L23:%.*]] = getelementptr i8, ptr [[PL]], i64 23
-; CHECK-NEXT: [[LOAD5:%.*]] = load i8, ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[LOAD7:%.*]] = load i8, ptr [[GEP_L1]], align 1
; CHECK-NEXT: [[LOAD14:%.*]] = load i8, ptr [[GEP_L14]], align 1
-; CHECK-NEXT: [[LOAD15:%.*]] = load i8, ptr [[GEP_S11]], align 1
-; CHECK-NEXT: [[LOAD20:%.*]] = load i8, ptr [[GEP_L20]], align 1
-; CHECK-NEXT: [[LOAD21:%.*]] = load i8, ptr [[GEP_L21]], align 1
-; CHECK-NEXT: [[LOAD22:%.*]] = load i8, ptr [[GEP_L22]], align 1
; CHECK-NEXT: [[LOAD23:%.*]] = load i8, ptr [[GEP_L23]], align 1
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[PS]], i64 2
; CHECK-NEXT: [[GEP_S9:%.*]] = getelementptr i8, ptr [[PS]], i64 4
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[PS]], i64 6
-; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[PS]], i64 8
-; CHECK-NEXT: [[GEP_S12:%.*]] = getelementptr i8, ptr [[PS]], i64 16
-; CHECK-NEXT: [[GEP_S13:%.*]] = getelementptr i8, ptr [[PS]], i64 18
-; CHECK-NEXT: [[GEP_S14:%.*]] = getelementptr i8, ptr [[PS]], i64 20
-; CHECK-NEXT: [[GEP_S15:%.*]] = getelementptr i8, ptr [[PS]], i64 22
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[PS]], i64 9
; CHECK-NEXT: [[GEP_S23:%.*]] = getelementptr i8, ptr [[PS]], i64 38
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = add <8 x i8> [[TMP1]], splat (i8 1)
+; CHECK-NEXT: [[TMP4:%.*]] = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 [[TMP5]], <8 x i1> splat (i1 true), <8 x i8> poison)
; CHECK-NEXT: [[TMP3:%.*]] = load <8 x i8>, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: store i8 [[LOAD5]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: store i8 [[LOAD7]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP9:%.*]] = add <16 x i8> [[TMP8]], <i8 0, i8 1, i8 1, i8 1, i8 1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+; CHECK-NEXT: [[TMP10:%.*]] = call <7 x i8> @llvm.masked.load.v7i8.p0(ptr align 1 [[GEP_L5]], <7 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>, <7 x i8> poison)
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <7 x i8> [[TMP10]], <7 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[TMP12:%.*]] = add <4 x i8> [[TMP11]], splat (i8 1)
+; CHECK-NEXT: [[TMP13:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[GEP_S0]], i64 38, <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v2i8.p0.i64(<2 x i8> [[TMP13]], ptr align 1 [[GEP_S5]], i64 2, <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: store i8 [[LOAD14]], ptr [[GEP_S9]], align 1
-; CHECK-NEXT: store i8 [[LOAD15]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: store <8 x i8> [[TMP2]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: store i8 [[LOAD20]], ptr [[GEP_S12]], align 1
-; CHECK-NEXT: store i8 [[LOAD21]], ptr [[GEP_S13]], align 1
-; CHECK-NEXT: store i8 [[LOAD22]], ptr [[GEP_S14]], align 1
-; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v8i8.p0.i64(<8 x i8> [[TMP3]], ptr align 1 [[GEP_S15]], i64 2, <8 x i1> splat (i1 true), i32 8)
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP12]], ptr align 1 [[GEP_S6]], i64 2, <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v16i8.p0.i64(<16 x i8> [[TMP9]], ptr align 1 [[GEP_S3]], i64 2, <16 x i1> splat (i1 true), i32 16)
; CHECK-NEXT: store i8 [[LOAD23]], ptr [[GEP_S23]], align 1
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
index ea5b9e14c8e0b6..a46449c3124e2b 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
@@ -87,16 +87,14 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; CHECK-NEXT: [[TMP66:%.*]] = zext <4 x i8> [[TMP65]] to <4 x i32>
; CHECK-NEXT: [[TMP67:%.*]] = sub <4 x i32> [[TMP64]], [[TMP66]]
; CHECK-NEXT: [[TMP68:%.*]] = shufflevector <4 x i32> [[TMP67]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[TMP69:%.*]] = insertelement <2 x i8> poison, i8 [[TMP115]], i64 0
-; CHECK-NEXT: [[TMP70:%.*]] = insertelement <2 x i8> [[TMP69]], i8 [[TMP0]], i64 1
+; CHECK-NEXT: [[TMP69:%.*]] = insertelement <4 x i8> poison, i8 [[TMP115]], i64 0
+; CHECK-NEXT: [[TMP70:%.*]] = insertelement <4 x i8> [[TMP69]], i8 [[TMP0]], i64 1
+; CHECK-NEXT: [[TMP117:%.*]] = shufflevector <2 x i8> [[TMP71]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP119:%.*]] = shufflevector <4 x i8> [[TMP70]], <4 x i8> [[TMP117]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[TMP72:%.*]] = zext <4 x i8> [[TMP119]] to <4 x i32>
; CHECK-NEXT: [[TMP73:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_3]], align 1
; CHECK-NEXT: [[TMP74:%.*]] = zext <4 x i8> [[TMP73]] to <4 x i32>
; CHECK-NEXT: [[TMP75:%.*]] = shufflevector <4 x i32> [[TMP74]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[TMP117:%.*]] = zext <2 x i8> [[TMP70]] to <2 x i32>
-; CHECK-NEXT: [[TMP119:%.*]] = shufflevector <2 x i32> [[TMP117]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP120:%.*]] = zext <2 x i8> [[TMP71]] to <2 x i32>
-; CHECK-NEXT: [[TMP121:%.*]] = shufflevector <2 x i32> [[TMP120]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP72:%.*]] = shufflevector <4 x i32> [[TMP119]], <4 x i32> [[TMP121]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; CHECK-NEXT: [[TMP76:%.*]] = sub <4 x i32> [[TMP72]], [[TMP75]]
; CHECK-NEXT: [[TMP77:%.*]] = shl <4 x i32> [[TMP76]], splat (i32 16)
; CHECK-NEXT: [[TMP78:%.*]] = add <4 x i32> [[TMP77]], [[TMP68]]
More information about the llvm-commits
mailing list