[llvm] [SLP] Pass along stride information when costing strided loads and stores (PR #227823)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 11:53:53 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Ryan Buchner (bababuck)
<details>
<summary>Changes</summary>
In cases of constant strides, can give a more accurate cost when accounting for a known stride.
Currently, only RISCV backend makes use of this information.
Assisted By: Codex
---
Full diff: https://github.com/llvm/llvm-project/pull/227823.diff
5 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+29-16)
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp (+12)
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h (+5)
- (modified) llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-stores.ll (+17-28)
- (modified) llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll (+5-7)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 30de440e44721..04f4dd912cd50 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14610,18 +14610,20 @@ void BoUpSLP::transformNodes() {
BaseLI->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(BaseLI->getPointerOperand())) +
getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, CostKind, Mask);
+ Type *StrideTy = DL->getIndexType(
+ cast<LoadInst>(E.Scalars.front())->getPointerOperand()->getType());
InstructionCost StridedCost = TTI->getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_load,
- VecTy, BaseLI->getPointerOperand(),
- /*VariableMask=*/false, CommonAlignment,
- BaseLI),
+ MemIntrinsicCostAttributes(
+ Intrinsic::experimental_vp_strided_load, VecTy,
+ BaseLI->getPointerOperand(),
+ /*VariableMask=*/false, CommonAlignment, BaseLI,
+ ConstantInt::getSigned(
+ StrideTy,
+ -static_cast<int64_t>(DL->getTypeAllocSize(ScalarTy)))),
CostKind);
if (StridedCost < OriginalVecCost || ForceStridedLoads) {
// Strided load is more profitable than consecutive load + reverse -
// transform the node to strided load.
- Type *StrideTy = DL->getIndexType(cast<LoadInst>(E.Scalars.front())
- ->getPointerOperand()
- ->getType());
StridedPtrInfo SPtrInfo;
SPtrInfo.StrideVal = ConstantInt::get(StrideTy, 1);
SPtrInfo.Ty = VecTy;
@@ -14651,19 +14653,21 @@ void BoUpSLP::transformNodes() {
BaseSI->getPointerAddressSpace(), CostKind,
getOperandInfo(E.getOperand(0))) +
getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, CostKind, Mask);
+ Type *StrideTy = DL->getIndexType(
+ cast<StoreInst>(E.Scalars.front())->getPointerOperand()->getType());
InstructionCost StridedCost = TTI->getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_store,
- VecTy, BaseSI->getPointerOperand(),
- /*VariableMask=*/false, CommonAlignment,
- BaseSI),
+ MemIntrinsicCostAttributes(
+ Intrinsic::experimental_vp_strided_store, VecTy,
+ BaseSI->getPointerOperand(),
+ /*VariableMask=*/false, CommonAlignment, BaseSI,
+ ConstantInt::getSigned(
+ StrideTy,
+ -static_cast<int64_t>(DL->getTypeAllocSize(ScalarTy)))),
CostKind);
if (StridedCost < OriginalVecCost) {
// Strided store is more profitable than reverse + consecutive store -
// transform the node to strided store.
E.State = TreeEntry::StridedVectorize;
- Type *StrideTy = DL->getIndexType(cast<StoreInst>(E.Scalars.front())
- ->getPointerOperand()
- ->getType());
StridedPtrInfo SPtrInfo;
SPtrInfo.StrideVal = ConstantInt::getSigned(StrideTy, -1);
SPtrInfo.Ty = VecTy;
@@ -17479,10 +17483,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
assert(StridedLoadTy && "Missing StridedPointerInfo for tree entry.");
Align CommonAlignment =
computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
+ Type *StrideTy = DL->getIndexType(LI0->getPointerOperand()->getType());
+ Value *Stride = getStrideBytesIfConstant(SPtrInfo.StrideVal, ScalarTy,
+ StrideTy, *DL);
VecLdCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_load,
StridedLoadTy, LI0->getPointerOperand(),
- /*VariableMask=*/false, CommonAlignment),
+ /*VariableMask=*/false, CommonAlignment,
+ /*I=*/nullptr, Stride),
CostKind);
if (StridedLoadTy != VecTy)
VecLdCost +=
@@ -17596,11 +17604,16 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
assert(StridedStoreTy && "Missing StridedPointerInfo for tree entry.");
Align CommonAlignment =
computeCommonAlignment<StoreInst>(UniqueValues.getArrayRef());
+ Type *StrideTy =
+ DL->getIndexType(BaseSI->getPointerOperand()->getType());
+ Value *Stride = getStrideBytesIfConstant(SPtrInfo.StrideVal, ScalarTy,
+ StrideTy, *DL);
VecStCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_store,
StridedStoreTy,
BaseSI->getPointerOperand(),
- /*VariableMask=*/false, CommonAlignment),
+ /*VariableMask=*/false, CommonAlignment,
+ /*I=*/nullptr, Stride),
CostKind);
if (StridedStoreTy != VecTy)
VecStCost +=
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
index 8279c277a1f04..7307f0956d5d8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
@@ -40,6 +40,18 @@ using namespace llvm;
namespace llvm::slpvectorizer {
+ConstantInt *getStrideBytesIfConstant(Value *Stride, Type *ScalarTy,
+ Type *StrideTy, const DataLayout &DL) {
+ auto *CI = dyn_cast_or_null<ConstantInt>(Stride);
+ if (!CI)
+ return nullptr;
+
+ unsigned StrideWidth = cast<IntegerType>(StrideTy)->getBitWidth();
+ APInt ElementSize(StrideWidth, DL.getTypeAllocSize(ScalarTy));
+ APInt StrideValue = CI->getValue().sextOrTrunc(StrideWidth) * ElementSize;
+ return ConstantInt::get(StrideTy->getContext(), StrideValue);
+}
+
bool arePointersCompatible(Value *Ptr1, Value *Ptr2,
const TargetLibraryInfo &TLI, unsigned MaxDepth,
bool CompareOpcodes) {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
index d1df27e1fb6c3..afb84e8150f64 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
@@ -22,6 +22,7 @@
namespace llvm {
class AssumptionCache;
class BasicBlock;
+class ConstantInt;
class DataLayout;
class DominatorTree;
class FixedVectorType;
@@ -40,6 +41,10 @@ bool arePointersCompatible(Value *Ptr1, Value *Ptr2,
const TargetLibraryInfo &TLI, unsigned MaxDepth,
bool CompareOpcodes = true);
+/// Returns the byte stride if \p Stride is a constant, or nullptr otherwise.
+ConstantInt *getStrideBytesIfConstant(Value *Stride, Type *ScalarTy,
+ Type *StrideTy, const DataLayout &DL);
+
/// Calculates minimal alignment as a common alignment.
template <typename T> Align computeCommonAlignment(ArrayRef<Value *> VL);
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-stores.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-stores.ll
index a738dd27bcaf7..192669794e11f 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-stores.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-stores.ll
@@ -528,45 +528,34 @@ define void @vf_ordering_issue(ptr %pl, ptr %ps) {
; CHECK-LABEL: define void @vf_ordering_issue(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 38
; CHECK-NEXT: [[GEP_L14:%.*]] = getelementptr i8, ptr [[PL]], i64 92
-; CHECK-NEXT: [[GEP_S11:%.*]] = getelementptr i8, ptr [[PL]], i64 33
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L20:%.*]] = getelementptr i8, ptr [[PL]], i64 13
-; CHECK-NEXT: [[GEP_L21:%.*]] = getelementptr i8, ptr [[PL]], i64 83
-; CHECK-NEXT: [[GEP_L22:%.*]] = getelementptr i8, ptr [[PL]], i64 32
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x ptr> poison, ptr [[PL]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x ptr> [[TMP1]], <8 x ptr> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, <8 x ptr> [[TMP2]], <8 x i64> <i64 33, i64 4, i64 6, i64 8, i64 10, i64 13, i64 83, i64 32>
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 15
; CHECK-NEXT: [[GEP_L23:%.*]] = getelementptr i8, ptr [[PL]], i64 23
-; CHECK-NEXT: [[LOAD5:%.*]] = load i8, ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[LOAD7:%.*]] = load i8, ptr [[GEP_L1]], align 1
; CHECK-NEXT: [[LOAD14:%.*]] = load i8, ptr [[GEP_L14]], align 1
-; CHECK-NEXT: [[LOAD15:%.*]] = load i8, ptr [[GEP_S11]], align 1
-; CHECK-NEXT: [[LOAD20:%.*]] = load i8, ptr [[GEP_L20]], align 1
-; CHECK-NEXT: [[LOAD21:%.*]] = load i8, ptr [[GEP_L21]], align 1
-; CHECK-NEXT: [[LOAD22:%.*]] = load i8, ptr [[GEP_L22]], align 1
; CHECK-NEXT: [[LOAD23:%.*]] = load i8, ptr [[GEP_L23]], align 1
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[PS]], i64 2
; CHECK-NEXT: [[GEP_S9:%.*]] = getelementptr i8, ptr [[PS]], i64 4
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[PS]], i64 6
-; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[PS]], i64 8
-; CHECK-NEXT: [[GEP_S12:%.*]] = getelementptr i8, ptr [[PS]], i64 16
-; CHECK-NEXT: [[GEP_S13:%.*]] = getelementptr i8, ptr [[PS]], i64 18
-; CHECK-NEXT: [[GEP_S14:%.*]] = getelementptr i8, ptr [[PS]], i64 20
-; CHECK-NEXT: [[GEP_S15:%.*]] = getelementptr i8, ptr [[PS]], i64 22
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[PS]], i64 9
; CHECK-NEXT: [[GEP_S23:%.*]] = getelementptr i8, ptr [[PS]], i64 38
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = add <8 x i8> [[TMP1]], splat (i8 1)
+; CHECK-NEXT: [[TMP4:%.*]] = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 [[TMP5]], <8 x i1> splat (i1 true), <8 x i8> poison)
; CHECK-NEXT: [[TMP3:%.*]] = load <8 x i8>, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: store i8 [[LOAD5]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: store i8 [[LOAD7]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP9:%.*]] = add <16 x i8> [[TMP8]], <i8 0, i8 1, i8 1, i8 1, i8 1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+; CHECK-NEXT: [[TMP10:%.*]] = call <7 x i8> @llvm.masked.load.v7i8.p0(ptr align 1 [[GEP_L5]], <7 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>, <7 x i8> poison)
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <7 x i8> [[TMP10]], <7 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[TMP12:%.*]] = add <4 x i8> [[TMP11]], splat (i8 1)
+; CHECK-NEXT: [[TMP13:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[GEP_S0]], i64 38, <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v2i8.p0.i64(<2 x i8> [[TMP13]], ptr align 1 [[GEP_S5]], i64 2, <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: store i8 [[LOAD14]], ptr [[GEP_S9]], align 1
-; CHECK-NEXT: store i8 [[LOAD15]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: store <8 x i8> [[TMP2]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: store i8 [[LOAD20]], ptr [[GEP_S12]], align 1
-; CHECK-NEXT: store i8 [[LOAD21]], ptr [[GEP_S13]], align 1
-; CHECK-NEXT: store i8 [[LOAD22]], ptr [[GEP_S14]], align 1
-; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v8i8.p0.i64(<8 x i8> [[TMP3]], ptr align 1 [[GEP_S15]], i64 2, <8 x i1> splat (i1 true), i32 8)
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP12]], ptr align 1 [[GEP_S6]], i64 2, <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v16i8.p0.i64(<16 x i8> [[TMP9]], ptr align 1 [[GEP_S3]], i64 2, <16 x i1> splat (i1 true), i32 16)
; CHECK-NEXT: store i8 [[LOAD23]], ptr [[GEP_S23]], align 1
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
index ea5b9e14c8e0b..a46449c3124e2 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
@@ -87,16 +87,14 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; CHECK-NEXT: [[TMP66:%.*]] = zext <4 x i8> [[TMP65]] to <4 x i32>
; CHECK-NEXT: [[TMP67:%.*]] = sub <4 x i32> [[TMP64]], [[TMP66]]
; CHECK-NEXT: [[TMP68:%.*]] = shufflevector <4 x i32> [[TMP67]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[TMP69:%.*]] = insertelement <2 x i8> poison, i8 [[TMP115]], i64 0
-; CHECK-NEXT: [[TMP70:%.*]] = insertelement <2 x i8> [[TMP69]], i8 [[TMP0]], i64 1
+; CHECK-NEXT: [[TMP69:%.*]] = insertelement <4 x i8> poison, i8 [[TMP115]], i64 0
+; CHECK-NEXT: [[TMP70:%.*]] = insertelement <4 x i8> [[TMP69]], i8 [[TMP0]], i64 1
+; CHECK-NEXT: [[TMP117:%.*]] = shufflevector <2 x i8> [[TMP71]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP119:%.*]] = shufflevector <4 x i8> [[TMP70]], <4 x i8> [[TMP117]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[TMP72:%.*]] = zext <4 x i8> [[TMP119]] to <4 x i32>
; CHECK-NEXT: [[TMP73:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_3]], align 1
; CHECK-NEXT: [[TMP74:%.*]] = zext <4 x i8> [[TMP73]] to <4 x i32>
; CHECK-NEXT: [[TMP75:%.*]] = shufflevector <4 x i32> [[TMP74]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[TMP117:%.*]] = zext <2 x i8> [[TMP70]] to <2 x i32>
-; CHECK-NEXT: [[TMP119:%.*]] = shufflevector <2 x i32> [[TMP117]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP120:%.*]] = zext <2 x i8> [[TMP71]] to <2 x i32>
-; CHECK-NEXT: [[TMP121:%.*]] = shufflevector <2 x i32> [[TMP120]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP72:%.*]] = shufflevector <4 x i32> [[TMP119]], <4 x i32> [[TMP121]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; CHECK-NEXT: [[TMP76:%.*]] = sub <4 x i32> [[TMP72]], [[TMP75]]
; CHECK-NEXT: [[TMP77:%.*]] = shl <4 x i32> [[TMP76]], splat (i32 16)
; CHECK-NEXT: [[TMP78:%.*]] = add <4 x i32> [[TMP77]], [[TMP68]]
``````````
</details>
https://github.com/llvm/llvm-project/pull/227823
More information about the llvm-commits
mailing list