[llvm] [SLP]Prefer segmented/deinterleaved loads to strided and fix codegen (PR #135058)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 10 10:01:14 PDT 2025
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/135058
>From 1fabde4b05ad586b155c1444fa287d0391f71fa0 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Wed, 9 Apr 2025 17:45:47 +0000
Subject: [PATCH 1/2] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.5
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 64 ++++++++++++-------
.../SLPVectorizer/RISCV/complex-loads.ll | 6 +-
.../RISCV/segmented-loads-simple.ll | 6 +-
.../X86/reorder-reused-masked-gather.ll | 2 +-
4 files changed, 49 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 87fc617eddedc..3ed4bd226b372 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5525,9 +5525,9 @@ static bool isMaskedLoadCompress(
// Check for very large distances between elements.
if (*Diff / Sz >= MaxRegSize / 8)
return false;
- Align CommonAlignment = computeCommonAlignment<LoadInst>(VL);
LoadVecTy = getWidenedType(ScalarTy, *Diff + 1);
auto *LI = cast<LoadInst>(Order.empty() ? VL.front() : VL[Order.front()]);
+ Align CommonAlignment = LI->getAlign();
IsMasked = !isSafeToLoadUnconditionally(
Ptr0, LoadVecTy, CommonAlignment, DL,
cast<LoadInst>(Order.empty() ? VL.back() : VL[Order.back()]), &AC, &DT,
@@ -5566,19 +5566,20 @@ static bool isMaskedLoadCompress(
TTI.getMaskedMemoryOpCost(Instruction::Load, LoadVecTy, CommonAlignment,
LI->getPointerAddressSpace(), CostKind);
} else {
- CommonAlignment = LI->getAlign();
LoadCost =
TTI.getMemoryOpCost(Instruction::Load, LoadVecTy, CommonAlignment,
LI->getPointerAddressSpace(), CostKind);
}
if (IsStrided) {
// Check for potential segmented(interleaved) loads.
- if (TTI.isLegalInterleavedAccessType(LoadVecTy, CompressMask[1],
+ auto *AlignedLoadVecTy = getWidenedType(
+ ScalarTy, getFullVectorNumberOfElements(TTI, ScalarTy, *Diff + 1));
+ if (TTI.isLegalInterleavedAccessType(AlignedLoadVecTy, CompressMask[1],
CommonAlignment,
LI->getPointerAddressSpace())) {
InstructionCost InterleavedCost =
VectorGEPCost + TTI.getInterleavedMemoryOpCost(
- Instruction::Load, LoadVecTy, CompressMask[1],
+ Instruction::Load, AlignedLoadVecTy, CompressMask[1],
std::nullopt, CommonAlignment,
LI->getPointerAddressSpace(), CostKind, IsMasked);
if (!Mask.empty())
@@ -5586,6 +5587,7 @@ static bool isMaskedLoadCompress(
VecTy, Mask, CostKind);
if (InterleavedCost < GatherCost) {
InterleaveFactor = CompressMask[1];
+ LoadVecTy = AlignedLoadVecTy;
return true;
}
}
@@ -5739,6 +5741,18 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
// Check that the sorted loads are consecutive.
if (static_cast<unsigned>(*Diff) == Sz - 1)
return LoadsState::Vectorize;
+ bool IsMasked;
+ unsigned InterleaveFactor;
+ SmallVector<int> CompressMask;
+ VectorType *LoadVecTy;
+ if (isMaskedLoadCompress(
+ VL, PointerOps, Order, *TTI, *DL, *SE, *AC, *DT, *TLI,
+ [&](Value *V) {
+ return areAllUsersVectorized(cast<Instruction>(V),
+ UserIgnoreList);
+ },
+ IsMasked, InterleaveFactor, CompressMask, LoadVecTy))
+ return LoadsState::CompressVectorize;
// Simple check if not a strided access - clear order.
bool IsPossibleStrided = *Diff % (Sz - 1) == 0;
// Try to generate strided load node.
@@ -5752,18 +5766,6 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
isStridedLoad(VL, PointerOps, Order, *TTI, *DL, *SE,
IsAnyPointerUsedOutGraph, *Diff))
return LoadsState::StridedVectorize;
- bool IsMasked;
- unsigned InterleaveFactor;
- SmallVector<int> CompressMask;
- VectorType *LoadVecTy;
- if (isMaskedLoadCompress(
- VL, PointerOps, Order, *TTI, *DL, *SE, *AC, *DT, *TLI,
- [&](Value *V) {
- return areAllUsersVectorized(cast<Instruction>(V),
- UserIgnoreList);
- },
- IsMasked, InterleaveFactor, CompressMask, LoadVecTy))
- return LoadsState::CompressVectorize;
}
if (!TTI->isLegalMaskedGather(VecTy, CommonAlignment) ||
TTI->forceScalarizeMaskedGather(VecTy, CommonAlignment))
@@ -17558,20 +17560,34 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
*TLI, [](Value *) { return true; }, IsMasked, InterleaveFactor,
CompressMask, LoadVecTy);
assert(IsVectorized && "Expected to be vectorized");
- Align CommonAlignment;
- if (IsMasked)
- CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
- else
- CommonAlignment = LI->getAlign();
+ Align CommonAlignment = LI->getAlign();
if (IsMasked) {
+ unsigned VF = getNumElements(LoadVecTy);
SmallVector<Constant *> MaskValues(
- getNumElements(LoadVecTy) / getNumElements(LI->getType()),
+ VF / getNumElements(LI->getType()),
ConstantInt::getFalse(VecTy->getContext()));
for (int I : CompressMask)
MaskValues[I] = ConstantInt::getTrue(VecTy->getContext());
Constant *MaskValue = ConstantVector::get(MaskValues);
- NewLI = Builder.CreateMaskedLoad(LoadVecTy, PO, CommonAlignment,
- MaskValue);
+ if (InterleaveFactor) {
+ // FIXME: codegen currently recognizes only vp.load, not
+ // masked.load, as segmented (deinterleaved) loads.
+ Value *Operands[] = {
+ PO, MaskValue,
+ Builder.getInt32(InterleaveFactor * (E->Scalars.size() - 1) +
+ 1)};
+ Type *Types[] = {LoadVecTy, Operands[0]->getType()};
+ CallInst *WideLoad =
+ Builder.CreateIntrinsic(Intrinsic::vp_load, Types, Operands,
+ nullptr, "wide.masked.load");
+ WideLoad->addParamAttr(
+ 0, Attribute::getWithAlignment(WideLoad->getContext(),
+ CommonAlignment));
+ NewLI = WideLoad;
+ } else {
+ NewLI = Builder.CreateMaskedLoad(LoadVecTy, PO, CommonAlignment,
+ MaskValue);
+ }
} else {
NewLI = Builder.CreateAlignedLoad(LoadVecTy, PO, CommonAlignment);
}
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
index 0f56862446a9d..a2ad021ee3f19 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
@@ -79,7 +79,8 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; CHECK-NEXT: [[TMP59:%.*]] = add <4 x i32> [[TMP57]], [[TMP58]]
; CHECK-NEXT: [[TMP60:%.*]] = sub <4 x i32> [[TMP57]], [[TMP58]]
; CHECK-NEXT: [[TMP61:%.*]] = shufflevector <4 x i32> [[TMP59]], <4 x i32> [[TMP60]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT: [[TMP62:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 null, i64 4, <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.vp.load.v8i8.p0(ptr align 1 null, <8 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, i32 5)
+; CHECK-NEXT: [[TMP62:%.*]] = shufflevector <8 x i8> [[WIDE_MASKED_LOAD]], <8 x i8> poison, <2 x i32> <i32 0, i32 4>
; CHECK-NEXT: [[TMP63:%.*]] = load <4 x i8>, ptr null, align 1
; CHECK-NEXT: [[TMP64:%.*]] = zext <4 x i8> [[TMP63]] to <4 x i32>
; CHECK-NEXT: [[TMP65:%.*]] = load <4 x i8>, ptr null, align 1
@@ -210,7 +211,8 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; THR15-NEXT: [[TMP59:%.*]] = add <4 x i32> [[TMP57]], [[TMP58]]
; THR15-NEXT: [[TMP60:%.*]] = sub <4 x i32> [[TMP57]], [[TMP58]]
; THR15-NEXT: [[TMP61:%.*]] = shufflevector <4 x i32> [[TMP59]], <4 x i32> [[TMP60]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; THR15-NEXT: [[TMP62:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 null, i64 4, <2 x i1> splat (i1 true), i32 2)
+; THR15-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.vp.load.v8i8.p0(ptr align 1 null, <8 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, i32 5)
+; THR15-NEXT: [[TMP62:%.*]] = shufflevector <8 x i8> [[WIDE_MASKED_LOAD]], <8 x i8> poison, <2 x i32> <i32 0, i32 4>
; THR15-NEXT: [[TMP63:%.*]] = load <4 x i8>, ptr null, align 1
; THR15-NEXT: [[TMP64:%.*]] = zext <4 x i8> [[TMP63]] to <4 x i32>
; THR15-NEXT: [[TMP65:%.*]] = load <4 x i8>, ptr null, align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/segmented-loads-simple.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/segmented-loads-simple.ll
index 0718cc25fd80c..c8d06ba1c8355 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/segmented-loads-simple.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/segmented-loads-simple.ll
@@ -5,7 +5,8 @@ define i32 @sum_of_abs_stride_2(ptr noalias %a, ptr noalias %b) {
; CHECK-LABEL: define i32 @sum_of_abs_stride_2
; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = call <8 x i8> @llvm.experimental.vp.strided.load.v8i8.p0.i64(ptr align 1 [[A]], i64 2, <8 x i1> splat (i1 true), i32 8)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.vp.load.v16i8.p0(ptr align 1 [[A]], <16 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>, i32 15)
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_LOAD]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i8> @llvm.abs.v8i8(<8 x i8> [[TMP0]], i1 false)
; CHECK-NEXT: [[TMP2:%.*]] = sext <8 x i8> [[TMP1]] to <8 x i32>
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP2]])
@@ -57,7 +58,8 @@ define i32 @sum_of_abs_stride_3(ptr noalias %a, ptr noalias %b) {
; CHECK-LABEL: define i32 @sum_of_abs_stride_3
; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = call <8 x i8> @llvm.experimental.vp.strided.load.v8i8.p0.i64(ptr align 1 [[A]], i64 3, <8 x i1> splat (i1 true), i32 8)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <32 x i8> @llvm.vp.load.v32i8.p0(ptr align 1 [[A]], <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, i32 22)
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <32 x i8> [[WIDE_MASKED_LOAD]], <32 x i8> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i8> @llvm.abs.v8i8(<8 x i8> [[TMP0]], i1 false)
; CHECK-NEXT: [[TMP2:%.*]] = sext <8 x i8> [[TMP1]] to <8 x i32>
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP2]])
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather.ll
index 9369a5962e643..7bb436b9543bf 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather.ll
@@ -4,7 +4,7 @@
define void @test(ptr noalias %0, ptr %p) {
; CHECK-LABEL: @test(
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP0:%.*]], i64 2
-; CHECK-NEXT: [[TMP3:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0(ptr [[P:%.*]], i32 4, <16 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <16 x float> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0(ptr [[P:%.*]], i32 16, <16 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <16 x float> poison)
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <16 x float> [[TMP3]], <16 x float> poison, <8 x i32> <i32 15, i32 4, i32 5, i32 0, i32 2, i32 6, i32 7, i32 8>
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x float> [[TMP3]], <16 x float> poison, <16 x i32> <i32 15, i32 4, i32 5, i32 15, i32 4, i32 5, i32 15, i32 0, i32 5, i32 2, i32 6, i32 7, i32 8, i32 6, i32 7, i32 8>
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <16 x float> [[TMP3]], <16 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float poison, float 0.000000e+00, float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 4, i32 24, i32 15, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
>From 780d8a2d075f3388708fcaa999a911edd82fce00 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Wed, 9 Apr 2025 17:50:12 +0000
Subject: [PATCH 2/2] Fix formatting
Created using spr 1.3.5
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3ed4bd226b372..909cbeb7e479f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5579,8 +5579,8 @@ static bool isMaskedLoadCompress(
LI->getPointerAddressSpace())) {
InstructionCost InterleavedCost =
VectorGEPCost + TTI.getInterleavedMemoryOpCost(
- Instruction::Load, AlignedLoadVecTy, CompressMask[1],
- std::nullopt, CommonAlignment,
+ Instruction::Load, AlignedLoadVecTy,
+ CompressMask[1], std::nullopt, CommonAlignment,
LI->getPointerAddressSpace(), CostKind, IsMasked);
if (!Mask.empty())
InterleavedCost += ::getShuffleCost(TTI, TTI::SK_PermuteSingleSrc,
More information about the llvm-commits
mailing list