[llvm] [SLP]Drop auxiliary subtrees to pass the VF=2 instruction-count veto (PR #224108)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 16 11:14:22 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/224108
The veto rejected the whole VF=2 tree when an optional splat gather
root or gathered loads subtree tipped the vector instruction count
over the scalar count. Drop the reused auxiliary subtrees while the
tree exceeds the budget; the surviving gathers re-materialize their
scalars.
Assisted-by: Cursor
>From 31f89f8303fce9a2be37b1c6f994383b134aed02 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Wed, 16 Sep 2026 11:14:09 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 131 +++++++++++++++---
.../splat-gather-subtree-inst-count.ll | 21 ++-
2 files changed, 120 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e21dbce1ea547..c7c4dd8a9dfcd 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2449,6 +2449,11 @@ class slpvectorizer::BoUpSLP {
/// getNumScalarInsts().
uint64_t getNumVectorInsts(bool HasTreeLoop);
+ /// Returns true if the instruction-count veto is skipped for a tree costed at
+ /// \p TreeCost because it contains a poor-throughput operation whose real
+ /// vector savings the count check does not model.
+ bool bypassesInstCountCheck(InstructionCost TreeCost) const;
+
/// Return information about the vector formed for the specified index
/// of a vector of (the same) instruction.
TargetTransformInfo::OperandValueInfo
@@ -12996,6 +13001,25 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
const TargetLibraryInfo &TLI,
const BoUpSLP &R);
+// A poor-throughput entry's real vector-vs-scalar savings (fdiv/frem/fsqrt)
+// are already folded into TreeCost like any other entry, including all
+// shuffle/insert/extract overhead elsewhere in the tree. So bypassing the
+// instruction-count veto just has to trust that already-complete TreeCost
+// instead of guessing at extra savings: heavy gather/shuffle overhead raises
+// TreeCost too, and will still block the bypass.
+bool BoUpSLP::bypassesInstCountCheck(InstructionCost TreeCost) const {
+ if (!VectorizePoorThroughput || TreeCost >= -SLPCostThreshold)
+ return false;
+ PoorThroughputOpCache Cache;
+ return any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &Ptr) {
+ const TreeEntry &TE = *Ptr;
+ return TE.hasState() && !DeletedNodes.contains(&TE) && !TE.isGather() &&
+ !TransformedToGatherNodes.contains(&TE) &&
+ TE.State != TreeEntry::CombinedVectorize &&
+ isPoorThroughputOp(TE.getMainOp(), *TTI, *TLI, Cache, CostKind);
+ });
+}
+
uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) {
uint64_t Total = 0;
for (const std::unique_ptr<TreeEntry> &Ptr : VectorizableTree) {
@@ -19020,6 +19044,85 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
if (!DeletedNodes.contains(BVE))
NodesCosts[BVE] = RecostEntry(BVE);
};
+ // The VF=2 instruction-count veto rejects the whole tree when the vector code
+ // has more instructions than the scalar code. The auxiliary subtrees - splat
+ // gather roots and gathered loads - are optional: the surviving gathers can
+ // re-materialize their scalars. Drop the reused ones while the tree exceeds
+ // the budget so a kept subtree cannot reject the main tree. The gate mirrors
+ // the veto conditions.
+ const bool ApplyInstCountVeto =
+ (!SplatGatheredScalarsRoots.empty() || !GatheredLoadsNodes.empty()) &&
+ CostKind != TTI::TCK_CodeSize && SLPInstCountCheck &&
+ TTI->preferSLPInstCountCheck() && getRootNode().getVectorFactor() == 2 &&
+ SLPCostThreshold == 0 &&
+ (!SLPReVec || !isa<VectorType>(getRootNodeScalars().front()->getType()));
+ const Loop *InstCountLoop = nullptr;
+ if (ApplyInstCountVeto && LoopAwareTripCount != 0 && getRootNode().hasState())
+ InstCountLoop = LI->getLoopFor(getRootNode().getMainOp()->getParent());
+ auto TrimAuxSubtreesForInstCount = [&](InstructionCost &Cost,
+ bool UseCurrentNodeCosts) {
+ if (!ApplyInstCountVeto || bypassesInstCountCheck(Cost))
+ return;
+ // Current cost of the subtree rooted at TE from the per-node costs.
+ auto GetCurrentSubtreeCost = [&](const TreeEntry *TE) {
+ InstructionCost C = NodesCosts.lookup(TE);
+ for (unsigned Idx : std::get<2>(SubtreeCosts[TE->Idx]))
+ C += NodesCosts.lookup(VectorizableTree[Idx].get());
+ return C;
+ };
+ auto DropReusedSubtree = [&](TreeEntry *TE, bool DeadOnly) {
+ if (DeletedNodes.contains(TE) || TransformedToGatherNodes.contains(TE))
+ return false;
+ ValuesToInsertTy ValuesToInsert;
+ const bool IsDead = FindDemandedElts(TE, ValuesToInsert).isZero();
+ if (!IsDead && DeadOnly)
+ return false;
+ InstructionCost CurrentGathersCost = 0;
+ for (const auto &[BVE, _] : ValuesToInsert)
+ CurrentGathersCost += NodesCosts.lookup(BVE);
+ DeletedNodes.insert(TE);
+ for (unsigned Idx : std::get<2>(SubtreeCosts[TE->Idx]))
+ DeletedNodes.insert(VectorizableTree[Idx].get());
+ // All reusers were dropped: the subtree is dead. Without current node
+ // costs the total conservatively keeps the subtree cost.
+ if (IsDead) {
+ if (UseCurrentNodeCosts)
+ Cost -= GetCurrentSubtreeCost(TE);
+ return true;
+ }
+ InstructionCost DroppedGathersCost = 0;
+ for (const auto &[BVE, _] : ValuesToInsert)
+ if (NodesCosts.contains(BVE))
+ DroppedGathersCost += RecostEntry(BVE);
+ // The pre-trimming subtree cost matches the total only while the tree
+ // is intact or restored; an accepted trimming leaves the total holding
+ // just the current costs of the surviving subtree nodes.
+ InstructionCost SubtreeCost = UseCurrentNodeCosts
+ ? GetCurrentSubtreeCost(TE)
+ : std::get<1>(SubtreeCosts[TE->Idx]);
+ Cost += DroppedGathersCost - CurrentGathersCost - SubtreeCost;
+ SyncGatherCosts(ValuesToInsert);
+ return true;
+ };
+ // Recompute the counts only after an actual drop. Dropping a root can
+ // kill the reusers of a root from the other list, so repeat until no more
+ // drops; dead subtrees are removed even under the budget.
+ uint64_t NumScalar = getNumScalarInsts(InstCountLoop);
+ uint64_t NumVector = getNumVectorInsts(InstCountLoop);
+ bool Retry;
+ do {
+ Retry = false;
+ for (ArrayRef<TreeEntry *> Roots :
+ {ArrayRef<TreeEntry *>(SplatGatheredScalarsRoots),
+ GatheredLoadsNodes.getArrayRef()})
+ for (TreeEntry *TE : Roots)
+ if (DropReusedSubtree(TE, /*DeadOnly=*/NumVector <= NumScalar)) {
+ NumScalar = getNumScalarInsts(InstCountLoop);
+ NumVector = getNumVectorInsts(InstCountLoop);
+ Retry = true;
+ }
+ } while (Retry);
+ };
if (!Changed) {
// The splat subtrees are not linked to the tree root, so their cost is
// not included in the root's subtree cost; add it explicitly. Drop the
@@ -19052,6 +19155,7 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
for (unsigned Idx : std::get<2>(SubtreeCosts[TE->Idx]))
DeletedNodes.insert(VectorizableTree[Idx].get());
}
+ TrimAuxSubtreesForInstCount(TotalCost, /*UseCurrentNodeCosts=*/false);
return TotalCost;
}
@@ -19151,8 +19255,10 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
<< ".\n"
<< "SLP: Current total cost = " << NewCost << "\n");
}
- if (NewCost + LoadsExtractsCost > Cost ||
- (!PreferTrimmedTree && NewCost + LoadsExtractsCost == Cost)) {
+ const bool Reverted =
+ NewCost + LoadsExtractsCost > Cost ||
+ (!PreferTrimmedTree && NewCost + LoadsExtractsCost == Cost);
+ if (Reverted) {
DeletedNodes.clear();
TransformedToGatherNodes.clear();
// The dropped splat subtrees stay deleted: they were excluded from the
@@ -19222,6 +19328,7 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
TransformedToGatherNodes.contains(VectorizableTree[2].get()))
return InstructionCost::getInvalid();
}
+ TrimAuxSubtreesForInstCount(NewCost, /*UseCurrentNodeCosts=*/!Reverted);
return NewCost;
}
@@ -19240,24 +19347,6 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
ArrayRef<Value *> VectorizedVals,
InstructionCost ReductionCost,
Instruction *RdxRoot) {
- // A poor-throughput entry's real vector-vs-scalar savings (fdiv/frem/fsqrt)
- // are already folded into TreeCost like any other entry, including all
- // shuffle/insert/extract overhead elsewhere in the tree. So bypassing the
- // instruction-count veto below just has to trust that already-complete
- // TreeCost instead of guessing at extra savings: heavy gather/shuffle
- // overhead raises TreeCost too, and will still block the bypass.
- auto BypassesInstCountCheck = [&]() {
- if (!VectorizePoorThroughput || TreeCost >= -SLPCostThreshold)
- return false;
- PoorThroughputOpCache Cache;
- return any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &Ptr) {
- const TreeEntry &TE = *Ptr;
- return TE.hasState() && !DeletedNodes.contains(&TE) && !TE.isGather() &&
- !TransformedToGatherNodes.contains(&TE) &&
- TE.State != TreeEntry::CombinedVectorize &&
- isPoorThroughputOp(TE.getMainOp(), *TTI, *TLI, Cache, CostKind);
- });
- };
// Reject vectorization if the vector code would produce more instructions
// than the scalar code. The cost model may underestimate overhead from
// shuffles, inserts, and extracts.
@@ -19277,7 +19366,7 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
uint64_t NumVector = getNumVectorInsts(TreeLoop);
LLVM_DEBUG(dbgs() << "SLP: Inst count check: vector=" << NumVector
<< " scalar=" << NumScalar << "\n");
- if (NumVector > NumScalar && !BypassesInstCountCheck()) {
+ if (NumVector > NumScalar && !bypassesInstCountCheck(TreeCost)) {
LLVM_DEBUG(dbgs() << "SLP: Rejecting tree: vector inst count "
<< NumVector << " > scalar inst count " << NumScalar
<< ".\n");
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/splat-gather-subtree-inst-count.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/splat-gather-subtree-inst-count.ll
index 5e043e5034e54..752f5c07b3d9a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/splat-gather-subtree-inst-count.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/splat-gather-subtree-inst-count.ll
@@ -9,18 +9,17 @@
define void @store_chain_splat_subtree(ptr %matrix, double %a) {
; CHECK-LABEL: @store_chain_splat_subtree(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP3:%.*]] = tail call double @llvm.fmuladd.f64(double [[A1:%.*]], double 0.000000e+00, double 0.000000e+00)
-; CHECK-NEXT: [[A:%.*]] = tail call double @llvm.fmuladd.f64(double [[A1]], double 0.000000e+00, double 0.000000e+00)
-; CHECK-NEXT: [[MUL0:%.*]] = fmul double [[A]], 0.000000e+00
-; CHECK-NEXT: [[TMP2:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP3]], double 0.000000e+00, double [[MUL0]])
-; CHECK-NEXT: [[TMP6:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP2]], double 0.000000e+00, double 0.000000e+00)
+; CHECK-NEXT: [[A:%.*]] = tail call double @llvm.fmuladd.f64(double [[A1:%.*]], double 0.000000e+00, double 0.000000e+00)
+; CHECK-NEXT: [[TMP1:%.*]] = tail call double @llvm.fmuladd.f64(double [[A1]], double 0.000000e+00, double 0.000000e+00)
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[MATRIX:%.*]], i64 24
-; CHECK-NEXT: store double [[TMP6]], ptr [[GEP0]], align 8
-; CHECK-NEXT: [[MUL1:%.*]] = fmul double [[A]], 0.000000e+00
-; CHECK-NEXT: [[TMP4:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP3]], double 0.000000e+00, double [[MUL1]])
-; CHECK-NEXT: [[TMP5:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP4]], double 0.000000e+00, double 0.000000e+00)
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[MATRIX]], i64 32
-; CHECK-NEXT: store double [[TMP5]], ptr [[GEP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x double> [[TMP3]], zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> poison, double [[A]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP6]], <2 x double> zeroinitializer, <2 x double> [[TMP4]])
+; CHECK-NEXT: [[TMP8:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP7]], <2 x double> zeroinitializer, <2 x double> zeroinitializer)
+; CHECK-NEXT: store <2 x double> [[TMP8]], ptr [[GEP0]], align 8
; CHECK-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list