[llvm] [SLP]Improve detection of gathered loads, if no other deps are detected. (PR #71967)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 10 09:58:02 PST 2023
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/71967
If the gather node includes ordered loads only partially (not the whole
node consists of loads) and the other gathered scalar are not loads, and
no other dependency from other nodes is found, we still can improve the
cost of gather, if take into account the fact that these loads still can
be vectorized.
>From 1bdcccad1b1642409a64e1556e9f07de2fa183c0 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Fri, 10 Nov 2023 08:10:01 -0800
Subject: [PATCH] [SLP]Improve detection of gathered loads, if no other deps
are detected.
If the gather node includes ordered loads only partially (not the whole
node consists of loads) and the other gathered scalar are not loads, and
no other dependency from other nodes is found, we still can improve the
cost of gather, if take into account the fact that these loads still can
be vectorized.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 64 +++++++++++++------
.../X86/remark-partial-loads-vectorize.ll | 2 +-
2 files changed, 44 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index bb233ed7d6c77ce..89225ef690b364d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7064,20 +7064,36 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
// Improve gather cost for gather of loads, if we can group some of the
// loads into vector loads.
InstructionsState S = getSameOpcode(VL, *R.TLI);
- if (VL.size() > 2 && S.getOpcode() == Instruction::Load &&
- !S.isAltShuffle() &&
+ const unsigned Sz = R.DL->getTypeSizeInBits(VL.front()->getType());
+ unsigned MinVF = R.getMinVF(2 * Sz);
+ if (VL.size() > 2 &&
+ ((S.getOpcode() == Instruction::Load && !S.isAltShuffle()) ||
+ (InVectors.empty() &&
+ any_of(seq<unsigned>(0, VL.size() / MinVF),
+ [&](unsigned Idx) {
+ ArrayRef<Value *> SubVL = VL.slice(Idx * MinVF, MinVF);
+ InstructionsState S = getSameOpcode(SubVL, *R.TLI);
+ return S.getOpcode() == Instruction::Load &&
+ !S.isAltShuffle();
+ }))) &&
!all_of(Gathers, [&](Value *V) { return R.getTreeEntry(V); }) &&
!isSplat(Gathers)) {
BoUpSLP::ValueSet VectorizedLoads;
+ SmallVector<LoadInst *> VectorizedStarts;
+ SmallVector<std::pair<unsigned, unsigned>> ScatterVectorized;
unsigned StartIdx = 0;
unsigned VF = VL.size() / 2;
- unsigned VectorizedCnt = 0;
- unsigned ScatterVectorizeCnt = 0;
- const unsigned Sz = R.DL->getTypeSizeInBits(S.MainOp->getType());
+ const unsigned Sz = R.DL->getTypeSizeInBits(VL.front()->getType());
for (unsigned MinVF = R.getMinVF(2 * Sz); VF >= MinVF; VF /= 2) {
for (unsigned Cnt = StartIdx, End = VL.size(); Cnt + VF <= End;
Cnt += VF) {
ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
+ if (S.getOpcode() != Instruction::Load || S.isAltShuffle()) {
+ InstructionsState SliceS = getSameOpcode(Slice, *R.TLI);
+ if (SliceS.getOpcode() != Instruction::Load ||
+ SliceS.isAltShuffle())
+ continue;
+ }
if (!VectorizedLoads.count(Slice.front()) &&
!VectorizedLoads.count(Slice.back()) && allSameBlock(Slice)) {
SmallVector<Value *> PointerOps;
@@ -7091,10 +7107,10 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
case LoadsState::PossibleStridedVectorize:
// Mark the vectorized loads so that we don't vectorize them
// again.
- if (LS == LoadsState::Vectorize)
- ++VectorizedCnt;
+ if (LS == LoadsState::Vectorize && CurrentOrder.empty())
+ VectorizedStarts.push_back(cast<LoadInst>(Slice.front()));
else
- ++ScatterVectorizeCnt;
+ ScatterVectorized.emplace_back(Cnt, VF);
VectorizedLoads.insert(Slice.begin(), Slice.end());
// If we vectorized initial block, no need to try to vectorize
// it again.
@@ -7125,8 +7141,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
// Exclude potentially vectorized loads from list of gathered
// scalars.
- auto *LI = cast<LoadInst>(S.MainOp);
- Gathers.assign(Gathers.size(), PoisonValue::get(LI->getType()));
+ Gathers.assign(Gathers.size(), PoisonValue::get(VL.front()->getType()));
// The cost for vectorized loads.
InstructionCost ScalarsCost = 0;
for (Value *V : VectorizedLoads) {
@@ -7136,17 +7151,24 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
LI->getAlign(), LI->getPointerAddressSpace(),
CostKind, TTI::OperandValueInfo(), LI);
}
- auto *LoadTy = FixedVectorType::get(LI->getType(), VF);
- Align Alignment = LI->getAlign();
- GatherCost +=
- VectorizedCnt *
- TTI.getMemoryOpCost(Instruction::Load, LoadTy, Alignment,
- LI->getPointerAddressSpace(), CostKind,
- TTI::OperandValueInfo(), LI);
- GatherCost += ScatterVectorizeCnt *
- TTI.getGatherScatterOpCost(
- Instruction::Load, LoadTy, LI->getPointerOperand(),
- /*VariableMask=*/false, Alignment, CostKind, LI);
+ auto *LoadTy = FixedVectorType::get(VL.front()->getType(), VF);
+ for (LoadInst *LI : VectorizedStarts) {
+ Align Alignment = LI->getAlign();
+ GatherCost +=
+ TTI.getMemoryOpCost(Instruction::Load, LoadTy, Alignment,
+ LI->getPointerAddressSpace(), CostKind,
+ TTI::OperandValueInfo(), LI);
+ }
+ for (std::pair<unsigned, unsigned> P : ScatterVectorized) {
+ auto *LI0 = cast<LoadInst>(VL[P.first]);
+ Align CommonAlignment = LI0->getAlign();
+ for (Value *V : VL.slice(P.first + 1, VF - 1))
+ CommonAlignment =
+ std::min(CommonAlignment, cast<LoadInst>(V)->getAlign());
+ GatherCost += TTI.getGatherScatterOpCost(
+ Instruction::Load, LoadTy, LI0->getPointerOperand(),
+ /*VariableMask=*/false, CommonAlignment, CostKind, LI0);
+ }
if (NeedInsertSubvectorAnalysis) {
// Add the cost for the subvectors insert.
for (int I = VF, E = VL.size(); I < E; I += VF)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark-partial-loads-vectorize.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark-partial-loads-vectorize.ll
index 84e155d837a336b..7de2cde45525ae2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/remark-partial-loads-vectorize.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark-partial-loads-vectorize.ll
@@ -8,7 +8,7 @@
; YAML-NEXT: Function: test
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'SLP vectorized with cost '
-; YAML-NEXT: - Cost: '-2'
+; YAML-NEXT: - Cost: '-4'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '4'
; YAML-LABEL: --- !Passed
More information about the llvm-commits
mailing list