[llvm] [SLP]Ignore the extracts in the instruction-count with high register pressure (PR #227864)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 13:50:28 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
In loops, the instruction-count check scales the tree entries by the
trip count, but not the extracts for the external users, so the extracts
just break the ties of the scaled counts.
With the fused fmul/fadd costing such ties reject the relaxation trees
of the reported kernel, the loop body stays scalar and spills. If more
values of the tree type are live in the block than there are registers,
ignore the extracts when they are the only reason to reject the tree.
Fixes the regression, reported in
https://github.com/llvm/llvm-project/pull/226117#issuecomment-5913566646
---
Full diff: https://github.com/llvm/llvm-project/pull/227864.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+43-4)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/inst-count-tie-reg-pressure.ll (+17-14)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index dbd27a3a53820..babe5988c7f32 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2506,9 +2506,9 @@ class slpvectorizer::BoUpSLP {
uint64_t getNumScalarInsts(bool HasTreeLoop);
/// Estimates the number of vector instructions (including buildvectors,
- /// shuffles, and extracts) the tree produces, weighted like
- /// getNumScalarInsts().
- uint64_t getNumVectorInsts(bool HasTreeLoop);
+ /// shuffles, and, if \p CountExtracts is set, extracts) the tree produces,
+ /// weighted like getNumScalarInsts().
+ uint64_t getNumVectorInsts(bool HasTreeLoop, bool CountExtracts = true);
/// Returns true if the instruction-count veto is skipped for a tree costed at
/// \p TreeCost because it contains a poor-throughput operation whose real
@@ -13384,7 +13384,7 @@ uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) {
return Total;
}
-uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) {
+uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop, bool CountExtracts) {
uint64_t Total = 0;
// Source vector -> max scale among the gather entries sharing it, so the
// combined shufflevector is still weighted like an in-loop entry below.
@@ -13463,6 +13463,8 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) {
}
for (const auto &VecAndScale : GatherExtractSourceVecs)
Total = SaturatingAdd(Total, VecAndScale.second);
+ if (!CountExtracts)
+ return Total;
// Count extract instructions from ExternalUses, skipping insertelements
// (those get folded into shuffles, not real extracts).
SmallPtrSet<Value *, 8> CountedExtracts;
@@ -20100,6 +20102,31 @@ template <typename T> struct ShuffledInsertData {
};
} // namespace
+/// \returns true if more values with the scalar type \p Ty than registers are
+/// live at some point of the block \p BB. The blocks larger than the scheduling
+/// budget are skipped to save compile time.
+static bool hasHighRegisterPressure(const BasicBlock &BB, Type *Ty,
+ const TargetTransformInfo &TTI) {
+ if (hasNItemsOrMore(BB, ScheduleRegionSizeBudget))
+ return false;
+ const unsigned NumRegs = TTI.getNumberOfRegisters(
+ TTI.getRegisterClassForType(/*Vector=*/false, Ty));
+ auto IsLiveCandidate = [&](const Value *V) {
+ return isa<Instruction, Argument>(V) && V->getType()->getScalarType() == Ty;
+ };
+ SmallPtrSet<const Value *, 32> Live;
+ for (const Instruction &I : reverse(BB)) {
+ // The operands of the PHIs are live in the predecessors.
+ if (isa<PHINode>(I))
+ break;
+ Live.erase(&I);
+ Live.insert_range(make_filter_range(I.operand_values(), IsLiveCandidate));
+ if (Live.size() > NumRegs)
+ return true;
+ }
+ return false;
+}
+
InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
ArrayRef<Value *> VectorizedVals,
InstructionCost ReductionCost,
@@ -20121,6 +20148,18 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
TreeLoop = LI->getLoopFor(getRootNode().getMainOp()->getParent());
uint64_t NumScalar = getNumScalarInsts(TreeLoop);
uint64_t NumVector = getNumVectorInsts(TreeLoop);
+ // In loops, the extracts are not scaled by the trip count and just break
+ // the ties of the scaled counts. Ignore them under the high register
+ // pressure: the vector code holds several lanes per register.
+ if (NumVector > NumScalar && TreeLoop && !ExternalUses.empty()) {
+ const uint64_t NumNoExtracts =
+ getNumVectorInsts(TreeLoop, /*CountExtracts=*/false);
+ Instruction *Root = getRootNode().getMainOp();
+ if (NumNoExtracts <= NumScalar &&
+ hasHighRegisterPressure(*Root->getParent(),
+ getValueType(Root, SLPReVec), *TTI))
+ NumVector = NumNoExtracts;
+ }
LLVM_DEBUG(dbgs() << "SLP: Inst count check: vector=" << NumVector
<< " scalar=" << NumScalar << "\n");
if (NumVector > NumScalar && !bypassesInstCountCheck(TreeCost)) {
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/inst-count-tie-reg-pressure.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/inst-count-tie-reg-pressure.ll
index 5af79e243814a..3fa95f30b21bb 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/inst-count-tie-reg-pressure.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/inst-count-tie-reg-pressure.ll
@@ -10,6 +10,9 @@ define void @test(ptr %src, ptr %dst, ptr %p0, ptr %p1, ptr %p2, ptr %p3, double
; CHECK-LABEL: define void @test(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], double [[U:%.*]], double [[V:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[U]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> <double poison, double 1.000000e+00>, double [[V]], i64 0
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[DST]], align 8
@@ -20,20 +23,16 @@ define void @test(ptr %src, ptr %dst, ptr %p0, ptr %p1, ptr %p2, ptr %p3, double
; CHECK-NEXT: [[L7:%.*]] = load double, ptr [[SRC]], align 8
; CHECK-NEXT: [[L8:%.*]] = load double, ptr [[DST]], align 8
; CHECK-NEXT: [[NEG_U:%.*]] = fneg double [[U]]
-; CHECK-NEXT: [[W_V:%.*]] = fmul double [[V]], f0x3FAC71C71C71C71C
-; CHECK-NEXT: [[U_1:%.*]] = fadd double [[U]], 1.000000e+00
; CHECK-NEXT: [[W2_U:%.*]] = fmul double [[U]], f0x3F9C71C71C71C71C
; CHECK-NEXT: [[V2_1:%.*]] = fmul double [[V]], [[V]]
; CHECK-NEXT: [[U_2:%.*]] = fadd double [[U]], 1.000000e+00
; CHECK-NEXT: [[W2_V:%.*]] = fmul double [[V]], f0x3F9C71C71C71C71C
-; CHECK-NEXT: [[W6_U:%.*]] = fmul double [[U]], f0x3FC5555555555555
; CHECK-NEXT: [[U2_0:%.*]] = fmul double [[U]], [[U]]
; CHECK-NEXT: [[W3_U_0:%.*]] = fmul double [[U]], f0x3FB5555555555555
; CHECK-NEXT: [[W3_U_1:%.*]] = fmul double [[U]], f0x3FB5555555555555
; CHECK-NEXT: [[U2_1:%.*]] = fmul double [[U]], [[U]]
; CHECK-NEXT: [[U2_2:%.*]] = fmul double [[U]], [[U]]
; CHECK-NEXT: [[U2_3:%.*]] = fmul double [[U]], [[U]]
-; CHECK-NEXT: [[H_1:%.*]] = fmul double [[U]], 5.000000e-01
; CHECK-NEXT: [[H_2:%.*]] = fmul double [[U]], 5.000000e-01
; CHECK-NEXT: [[H_3:%.*]] = fmul double [[U]], 5.000000e-01
; CHECK-NEXT: [[H_4:%.*]] = fmul double [[U]], 5.000000e-01
@@ -44,17 +43,21 @@ define void @test(ptr %src, ptr %dst, ptr %p0, ptr %p1, ptr %p2, ptr %p3, double
; CHECK-NEXT: [[F1:%.*]] = load double, ptr [[SRC]], align 8
; CHECK-NEXT: [[D0:%.*]] = fsub double [[F0]], [[F1]]
; CHECK-NEXT: [[ODD0:%.*]] = fsub double 0.000000e+00, [[U2_0]]
-; CHECK-NEXT: [[FEQ0:%.*]] = fmul reassoc contract double [[U_1]], [[W_V]]
-; CHECK-NEXT: [[EVEN0:%.*]] = fsub contract double [[H_1]], [[FEQ0]]
-; CHECK-NEXT: [[EVEN0_NEG:%.*]] = fmul contract double [[EVEN0]], -1.950000e+00
-; CHECK-NEXT: [[TMP12:%.*]] = fadd contract double [[EVEN0_NEG]], [[U]]
-; CHECK-NEXT: store double [[TMP12]], ptr [[SRC]], align 8
-; CHECK-NEXT: [[EVEN1:%.*]] = fsub double [[H_2]], [[W_V]]
; CHECK-NEXT: [[D1:%.*]] = fsub double [[L0]], [[F2]]
-; CHECK-NEXT: [[FA1:%.*]] = fmul double [[D1]], 5.000000e-01
-; CHECK-NEXT: [[ODD1:%.*]] = fsub double [[FA1]], [[W6_U]]
-; CHECK-NEXT: [[ODD1_NEG:%.*]] = fmul double [[ODD1]], f0xBFB0ECF56BE69CA7
-; CHECK-NEXT: [[TMP14:%.*]] = fadd double 0.000000e+00, [[ODD1_NEG]]
+; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP2]], <double f0x3FAC71C71C71C71C, double f0x3FC5555555555555>
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], <double 1.000000e+00, double -0.000000e+00>
+; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[TMP4]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x double> [[TMP1]], double [[D1]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = fmul <2 x double> [[TMP6]], splat (double 5.000000e-01)
+; CHECK-NEXT: [[TMP8:%.*]] = fsub <2 x double> [[TMP7]], [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = fmul <2 x double> [[TMP8]], <double -1.950000e+00, double f0xBFB0ECF56BE69CA7>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> <double poison, double 0.000000e+00>, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP11:%.*]] = fadd <2 x double> [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP11]], i64 0
+; CHECK-NEXT: store double [[TMP12]], ptr [[SRC]], align 8
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
+; CHECK-NEXT: [[EVEN1:%.*]] = fsub double [[H_2]], [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[TMP11]], i64 1
; CHECK-NEXT: store double [[TMP14]], ptr [[DST]], align 8
; CHECK-NEXT: [[V2_0:%.*]] = fmul double [[V]], [[V]]
; CHECK-NEXT: [[E2:%.*]] = fsub double [[H_3]], [[V2_0]]
``````````
</details>
https://github.com/llvm/llvm-project/pull/227864
More information about the llvm-commits
mailing list