[llvm] [SLP]Exclude splat gather subtrees from the store chain retry size (PR #221241)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 9 05:09:57 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/221241
>From 9024ba7039ad55b0d21df56e7cb0f9ede134aeaa Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Fri, 4 Sep 2026 07:55:17 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 13 ++++++++++++-
.../splat-gather-subtree-store-chain.ll | 19 +++++++------------
2 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 108b8a52d54bc..4eaee688180ce 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -794,6 +794,7 @@ class slpvectorizer::BoUpSLP {
IsGraphTransformMode = false;
GatheredLoadsEntriesFirst.reset();
SplatGatheredScalarsRoots.clear();
+ NumSplatSubtreeEntries = 0;
CompressEntryToData.clear();
ExternalUses.clear();
ExternalUsesAsOriginalScalar.clear();
@@ -827,6 +828,9 @@ class slpvectorizer::BoUpSLP {
/// Returns the base graph size, before any transformations.
unsigned getCanonicalGraphSize() const { return BaseGraphSize; }
+ /// Number of tree entries that form the splat gather subtrees.
+ unsigned getNumSplatSubtreeEntries() const { return NumSplatSubtreeEntries; }
+
/// Perform LICM and CSE on the newly generated gather sequences.
void optimizeGatherSequence();
@@ -3634,6 +3638,11 @@ class slpvectorizer::BoUpSLP {
/// before the root node.
SmallVector<TreeEntry *> SplatGatheredScalarsRoots;
+ /// Number of tree entries added while building the splat gather subtrees.
+ /// The subtrees are auxiliary and must not inflate the tree size recorded
+ /// for failed store chain attempts.
+ unsigned NumSplatSubtreeEntries = 0;
+
/// Maps compress entries to their mask data for the final codegen.
SmallDenseMap<const TreeEntry *,
std::tuple<SmallVector<int>, VectorType *, unsigned, bool>>
@@ -12065,6 +12074,7 @@ class InstructionsCompatibilityAnalysis {
} // namespace
void BoUpSLP::tryToVectorizeSplatGatheredScalars() {
+ unsigned PrevTreeSize = VectorizableTree.size();
auto LoadsSubkey = [](size_t /*Key*/, LoadInst *LI) {
return hash_value(getUnderlyingObject(LI->getPointerOperand()));
};
@@ -12136,6 +12146,7 @@ void BoUpSLP::tryToVectorizeSplatGatheredScalars() {
};
BuildSubtree(Groups);
BuildSubtree(FallbackGroups);
+ NumSplatSubtreeEntries = VectorizableTree.size() - PrevTreeSize;
}
BoUpSLP::ScalarsVectorizationLegality
@@ -29062,7 +29073,7 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable();
R.buildExternalUses();
- Size = R.getCanonicalGraphSize();
+ Size = R.getCanonicalGraphSize() - R.getNumSplatSubtreeEntries();
if (S && S.getOpcode() == Instruction::Load)
Size = 2; // cut off masked gather small trees
InstructionCost Cost = R.getTreeCost(TreeCost);
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/splat-gather-subtree-store-chain.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/splat-gather-subtree-store-chain.ll
index de820b4a0682a..afc7ca875f899 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/splat-gather-subtree-store-chain.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/splat-gather-subtree-store-chain.ll
@@ -12,25 +12,20 @@ define void @test(ptr %matrix, double %0) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[TMP1:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP0]], double 0.000000e+00, double 0.000000e+00)
-; CHECK-NEXT: [[TMP10:%.*]] = fadd double [[TMP1]], 0.000000e+00
-; CHECK-NEXT: [[TMP3:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP10]], double 0.000000e+00, double 0.000000e+00)
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[MATRIX]], i64 832
-; CHECK-NEXT: store double [[TMP3]], ptr [[GEP0]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP0]], double 0.000000e+00, double 0.000000e+00)
+; CHECK-NEXT: [[TMP3:%.*]] = fadd double [[TMP1]], 0.000000e+00
; CHECK-NEXT: [[TMP4:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP0]], double 0.000000e+00, double [[TMP2]])
-; CHECK-NEXT: [[TMP11:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP4]], double 0.000000e+00, double 0.000000e+00)
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[MATRIX]], i64 840
-; CHECK-NEXT: store double [[TMP11]], ptr [[GEP1]], align 8
; CHECK-NEXT: [[TMP5:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP4]], double [[TMP2]], double 0.000000e+00)
; CHECK-NEXT: [[TMP6:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP1]], double 0.000000e+00, double [[TMP5]])
-; CHECK-NEXT: [[TMP9:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP6]], double 0.000000e+00, double 0.000000e+00)
-; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr [[MATRIX]], i64 848
-; CHECK-NEXT: store double [[TMP9]], ptr [[GEP2]], align 8
; CHECK-NEXT: [[TMP7:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP4]], double 0.000000e+00, double 0.000000e+00)
; CHECK-NEXT: [[TMP8:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP1]], double 0.000000e+00, double [[TMP7]])
-; CHECK-NEXT: [[TMP12:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP8]], double 0.000000e+00, double 0.000000e+00)
-; CHECK-NEXT: [[GEP3:%.*]] = getelementptr i8, ptr [[MATRIX]], i64 856
-; CHECK-NEXT: store double [[TMP12]], ptr [[GEP3]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> poison, double [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP4]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP10]], double [[TMP6]], i64 2
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP8]], i64 3
+; CHECK-NEXT: [[TMP13:%.*]] = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> [[TMP12]], <4 x double> zeroinitializer, <4 x double> zeroinitializer)
+; CHECK-NEXT: store <4 x double> [[TMP13]], ptr [[GEP0]], align 8
; CHECK-NEXT: br label %[[LOOP]]
;
entry:
More information about the llvm-commits
mailing list