[llvm] [SLP]Fix regressions after the fmul/fadd fusion cost changes (PR #228720)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 08:02:41 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/228720
Revisit the standalone seeds skipped as dependent on the lanes of a
vectorized window, instead of dropping them until the next SLP run.
Do not credit the fusion of the root of a loop-carried fadd reduction:
the chain through the loop phi is bound by its latency, which the
vector code cuts.
Fixes regressions in RAJAPerf.
Assisted-by: Cursor
>From 6ceeb2c94a262b2a4b9f5f76e38193f50e1210e9 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 3 Oct 2026 08:02:28 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 30 ++++++++++++-----
.../AArch64/loop-accumulator-reduction.ll | 32 +++++++++----------
.../SLPVectorizer/X86/dependent-seeds.ll | 7 ++--
3 files changed, 41 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 0981c124e3aec..15b6e832c54be 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30649,8 +30649,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
SmallVector<Value *> Ops(ActualVF, nullptr);
unsigned Idx = 0;
SmallPtrSet<const Value *, 8> Taken;
- bool DependentSkipped = false;
- for (Value *V : VL.drop_front(I)) {
+ std::optional<unsigned> FirstSkipped;
+ for (auto [Off, V] : enumerate(VL.drop_front(I))) {
// Check that a previous iteration of this loop did not delete the
// Value.
auto *Inst = dyn_cast<Instruction>(V);
@@ -30661,7 +30661,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
if (StandaloneSeeds && Inst &&
any_of(Inst->operand_values(),
[&](const Value *Op) { return Taken.contains(Op); })) {
- DependentSkipped = true;
+ if (!FirstSkipped)
+ FirstSkipped = I + Off;
continue;
}
Ops[Idx] = V;
@@ -30674,7 +30675,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
// Not enough vectorizable instructions - exit. A skipped dependent seed
// may start the next window, so the scan must not stop because of it.
if (Idx != ActualVF) {
- if (!DependentSkipped)
+ if (!FirstSkipped)
break;
continue;
}
@@ -30727,8 +30728,9 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
<< ore::NV("TreeSize", R.getTreeSize()));
R.vectorizeTree();
- // Move to the next bundle.
- I += VF - 1;
+ // Move to the next bundle. The skipped dependent seeds are fed by the
+ // vectorized lanes now, the next bundle starts from the first of them.
+ I = FirstSkipped ? *FirstSkipped - 1 : I + VF - 1;
NextInst = I + 1;
Changed = true;
} else {
@@ -34004,6 +34006,15 @@ class HorizontalReduction {
return cast<Instruction>(U);
return cast<Instruction>(RdxVal);
};
+ // The root is accumulated through the phi of its loop.
+ const bool IsLoopCarriedRoot =
+ RdxKind == RecurKind::FAdd &&
+ any_of(ReductionRoot->users(), [&](User *U) {
+ auto *Phi = dyn_cast<PHINode>(U);
+ return Phi &&
+ DT.dominates(Phi->getParent(),
+ cast<Instruction>(ReductionRoot)->getParent());
+ });
auto EvaluateScalarCost =
[&](function_ref<InstructionCost(Instruction *)> GenCostFn) {
InstructionCost Cost = 0;
@@ -34027,8 +34038,11 @@ class HorizontalReduction {
for (User *U : RdxVal->users()) {
auto *RdxOp = cast<Instruction>(U);
// The reduction root is used outside of the reduction any
- // number of times, its fmul operand is still fused into it.
- if ((RdxKind == RecurKind::FAdd && RdxOp == ReductionRoot) ||
+ // number of times, its fmul operand is still fused into it. The
+ // loop-carried chain is bound by its latency, so the fusion does
+ // not make it cheaper than the vector code, which cuts the chain.
+ if ((RdxKind == RecurKind::FAdd && RdxOp == ReductionRoot &&
+ !IsLoopCarriedRoot) ||
hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
InstructionsState RdxOpS = RdxKind == RecurKind::FAdd
? getSameOpcode(RdxOp, TLI)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
index 41b197a68e4d0..15223e0bbe69b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
@@ -1359,12 +1359,12 @@ define double @loop_acc_fmul_chain(ptr %a, ptr %b, i64 %n, i64 %na, i64 %nb, dou
; CHECK-LABEL: define double @loop_acc_fmul_chain(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]], i64 [[NA:%.*]], i64 [[NB:%.*]], double [[ALPHA:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[ALPHA]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> <double poison, double 1.000000e+00, double poison, double poison>, double [[ALPHA]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[K:%.*]] = phi i64 [ [[K_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[DOT1:%.*]] = phi double [ [[OP_RDX1:%.*]], %[[LOOP]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: [[DOT:%.*]] = phi double [ [[OP_RDX1:%.*]], %[[LOOP]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: [[AM0:%.*]] = mul nuw nsw i64 [[K]], [[NA]]
; CHECK-NEXT: [[PA0:%.*]] = getelementptr double, ptr [[A]], i64 [[AM0]]
; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[PA0]], align 8
@@ -1373,32 +1373,32 @@ define double @loop_acc_fmul_chain(ptr %a, ptr %b, i64 %n, i64 %na, i64 %nb, dou
; CHECK-NEXT: [[PB0:%.*]] = getelementptr double, ptr [[B]], i64 [[BM0]]
; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[PB0]], align 8
; CHECK-NEXT: [[PROD0:%.*]] = fmul fast double [[SCALED0]], [[B0]]
-; CHECK-NEXT: [[ADD0:%.*]] = fadd fast double [[PROD0]], [[DOT1]]
; CHECK-NEXT: [[K1:%.*]] = add nuw nsw i64 [[K]], 1
; CHECK-NEXT: [[PA1:%.*]] = getelementptr double, ptr [[A]], i64 [[K1]]
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[PA1]], align 8
-; CHECK-NEXT: [[SCALED1:%.*]] = fmul fast double [[A1]], [[ALPHA]]
; CHECK-NEXT: [[BM1:%.*]] = mul nuw nsw i64 [[K1]], [[NB]]
; CHECK-NEXT: [[PB1:%.*]] = getelementptr double, ptr [[B]], i64 [[BM1]]
; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[PB1]], align 8
-; CHECK-NEXT: [[PROD1:%.*]] = fmul fast double [[SCALED1]], [[B1]]
-; CHECK-NEXT: [[ADD1:%.*]] = fadd fast double [[PROD1]], [[ADD0]]
; CHECK-NEXT: [[K2:%.*]] = add nuw nsw i64 [[K]], 2
-; CHECK-NEXT: [[PA2:%.*]] = getelementptr double, ptr [[A]], i64 [[K2]]
; CHECK-NEXT: [[BM2:%.*]] = mul nuw nsw i64 [[K2]], [[NB]]
; CHECK-NEXT: [[PB2:%.*]] = getelementptr double, ptr [[B]], i64 [[BM2]]
; CHECK-NEXT: [[B2:%.*]] = load double, ptr [[PB2]], align 8
; CHECK-NEXT: [[K3:%.*]] = add nuw nsw i64 [[K]], 3
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PA2]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = fmul fast <2 x double> [[TMP2]], [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
-; CHECK-NEXT: [[PROD2:%.*]] = fmul fast double [[TMP4]], [[B2]]
-; CHECK-NEXT: [[DOT:%.*]] = fadd fast double [[PROD2]], [[ADD1]]
+; CHECK-NEXT: [[PA3:%.*]] = getelementptr double, ptr [[A]], i64 [[K3]]
+; CHECK-NEXT: [[A3:%.*]] = load double, ptr [[PA3]], align 8
; CHECK-NEXT: [[BM3:%.*]] = mul nuw nsw i64 [[K3]], [[NB]]
; CHECK-NEXT: [[PB3:%.*]] = getelementptr double, ptr [[B]], i64 [[BM3]]
; CHECK-NEXT: [[B3:%.*]] = load double, ptr [[PB3]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
-; CHECK-NEXT: [[OP_RDX:%.*]] = fmul fast double [[TMP5]], [[B3]]
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PA1]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[A3]], i64 2
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP4]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast <4 x double> [[TMP5]], [[TMP1]]
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 0.000000e+00>, double [[B1]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x double> [[TMP7]], double [[B2]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> [[TMP8]], double [[B3]], i64 2
+; CHECK-NEXT: [[TMP10:%.*]] = fmul fast <4 x double> [[TMP9]], [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP10]])
+; CHECK-NEXT: [[OP_RDX:%.*]] = fadd fast double [[TMP11]], [[PROD0]]
; CHECK-NEXT: [[OP_RDX1]] = fadd fast double [[OP_RDX]], [[DOT]]
; CHECK-NEXT: [[K_NEXT]] = add nuw nsw i64 [[K]], 4
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/dependent-seeds.ll b/llvm/test/Transforms/SLPVectorizer/X86/dependent-seeds.ll
index db2e8017e3c27..b97a62c3b2538 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/dependent-seeds.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/dependent-seeds.ll
@@ -34,12 +34,11 @@ define double @second_level_seeds(ptr noalias %x, ptr noalias %y, ptr noalias %m
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[X]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = fadd fast double [[TMP4]], 1.000000e+00
+; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <2 x double> [[TMP3]], splat (double 1.000000e+00)
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i64 0
; CHECK-NEXT: store double [[TMP5]], ptr [[M]], align 8
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
-; CHECK-NEXT: [[TMP6:%.*]] = fadd fast double [[TMP7]], 1.000000e+00
; CHECK-NEXT: [[L:%.*]] = load double, ptr [[M]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i64 1
; CHECK-NEXT: [[R:%.*]] = fadd double [[TMP6]], [[L]]
; CHECK-NEXT: ret double [[R]]
;
More information about the llvm-commits
mailing list