[llvm] b11606a - [SLP]Fix regressions after the fmul/fadd fusion cost changes
via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 4 03:39:02 PDT 2026
Author: Alexey Bataev
Date: 2026-10-04T06:38:51-04:00
New Revision: b11606a912ae297cc4b9db6a7c59f44b0e326aa6
URL: https://github.com/llvm/llvm-project/commit/b11606a912ae297cc4b9db6a7c59f44b0e326aa6
DIFF: https://github.com/llvm/llvm-project/commit/b11606a912ae297cc4b9db6a7c59f44b0e326aa6.diff
LOG: [SLP]Fix regressions after the fmul/fadd fusion cost changes
Revisit the standalone seeds skipped as dependent on the lanes of a
vectorized window, instead of dropping them until the next SLP run.
Do not credit the fusion of the root of a loop-carried fadd reduction:
the chain through the loop phi is bound by its latency, which the
vector code cuts.
Fixes regressions in RAJAPerf.
Assisted-by: Cursor
Reviewers: RKSimon, bababuck
Pull Request: https://github.com/llvm/llvm-project/pull/228720
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
llvm/test/Transforms/SLPVectorizer/X86/dependent-seeds.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a03dfe49a85f1..c105f41bc15ab 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30649,8 +30649,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
SmallVector<Value *> Ops(ActualVF, nullptr);
unsigned Idx = 0;
SmallPtrSet<const Value *, 8> Taken;
- bool DependentSkipped = false;
- for (Value *V : VL.drop_front(I)) {
+ std::optional<unsigned> FirstSkipped;
+ for (auto [Off, V] : enumerate(VL.drop_front(I))) {
// Check that a previous iteration of this loop did not delete the
// Value.
auto *Inst = dyn_cast<Instruction>(V);
@@ -30661,7 +30661,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
if (StandaloneSeeds && Inst &&
any_of(Inst->operand_values(),
[&](const Value *Op) { return Taken.contains(Op); })) {
- DependentSkipped = true;
+ if (!FirstSkipped)
+ FirstSkipped = I + Off;
continue;
}
Ops[Idx] = V;
@@ -30674,7 +30675,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
// Not enough vectorizable instructions - exit. A skipped dependent seed
// may start the next window, so the scan must not stop because of it.
if (Idx != ActualVF) {
- if (!DependentSkipped)
+ if (!FirstSkipped)
break;
continue;
}
@@ -30727,8 +30728,9 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
<< ore::NV("TreeSize", R.getTreeSize()));
R.vectorizeTree();
- // Move to the next bundle.
- I += VF - 1;
+ // Move to the next bundle. The skipped dependent seeds are fed by the
+ // vectorized lanes now, the next bundle starts from the first of them.
+ I = FirstSkipped ? *FirstSkipped - 1 : I + VF - 1;
NextInst = I + 1;
Changed = true;
} else {
@@ -34009,6 +34011,15 @@ class HorizontalReduction {
return cast<Instruction>(U);
return cast<Instruction>(RdxVal);
};
+ // The root is accumulated through the phi of its loop.
+ const bool IsLoopCarriedRoot =
+ RdxKind == RecurKind::FAdd &&
+ any_of(ReductionRoot->users(), [&](User *U) {
+ auto *Phi = dyn_cast<PHINode>(U);
+ return Phi &&
+ DT.dominates(Phi->getParent(),
+ cast<Instruction>(ReductionRoot)->getParent());
+ });
auto EvaluateScalarCost =
[&](function_ref<InstructionCost(Instruction *)> GenCostFn) {
InstructionCost Cost = 0;
@@ -34032,8 +34043,11 @@ class HorizontalReduction {
for (User *U : RdxVal->users()) {
auto *RdxOp = cast<Instruction>(U);
// The reduction root is used outside of the reduction any
- // number of times, its fmul operand is still fused into it.
- if ((RdxKind == RecurKind::FAdd && RdxOp == ReductionRoot) ||
+ // number of times, its fmul operand is still fused into it. The
+ // loop-carried chain is bound by its latency, so the fusion does
+ // not make it cheaper than the vector code, which cuts the chain.
+ if ((RdxKind == RecurKind::FAdd && RdxOp == ReductionRoot &&
+ !IsLoopCarriedRoot) ||
hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
InstructionsState RdxOpS = RdxKind == RecurKind::FAdd
? getSameOpcode(RdxOp, TLI)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
index 41b197a68e4d0..15223e0bbe69b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
@@ -1359,12 +1359,12 @@ define double @loop_acc_fmul_chain(ptr %a, ptr %b, i64 %n, i64 %na, i64 %nb, dou
; CHECK-LABEL: define double @loop_acc_fmul_chain(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]], i64 [[NA:%.*]], i64 [[NB:%.*]], double [[ALPHA:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[ALPHA]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> <double poison, double 1.000000e+00, double poison, double poison>, double [[ALPHA]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[K:%.*]] = phi i64 [ [[K_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[DOT1:%.*]] = phi double [ [[OP_RDX1:%.*]], %[[LOOP]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: [[DOT:%.*]] = phi double [ [[OP_RDX1:%.*]], %[[LOOP]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: [[AM0:%.*]] = mul nuw nsw i64 [[K]], [[NA]]
; CHECK-NEXT: [[PA0:%.*]] = getelementptr double, ptr [[A]], i64 [[AM0]]
; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[PA0]], align 8
@@ -1373,32 +1373,32 @@ define double @loop_acc_fmul_chain(ptr %a, ptr %b, i64 %n, i64 %na, i64 %nb, dou
; CHECK-NEXT: [[PB0:%.*]] = getelementptr double, ptr [[B]], i64 [[BM0]]
; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[PB0]], align 8
; CHECK-NEXT: [[PROD0:%.*]] = fmul fast double [[SCALED0]], [[B0]]
-; CHECK-NEXT: [[ADD0:%.*]] = fadd fast double [[PROD0]], [[DOT1]]
; CHECK-NEXT: [[K1:%.*]] = add nuw nsw i64 [[K]], 1
; CHECK-NEXT: [[PA1:%.*]] = getelementptr double, ptr [[A]], i64 [[K1]]
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[PA1]], align 8
-; CHECK-NEXT: [[SCALED1:%.*]] = fmul fast double [[A1]], [[ALPHA]]
; CHECK-NEXT: [[BM1:%.*]] = mul nuw nsw i64 [[K1]], [[NB]]
; CHECK-NEXT: [[PB1:%.*]] = getelementptr double, ptr [[B]], i64 [[BM1]]
; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[PB1]], align 8
-; CHECK-NEXT: [[PROD1:%.*]] = fmul fast double [[SCALED1]], [[B1]]
-; CHECK-NEXT: [[ADD1:%.*]] = fadd fast double [[PROD1]], [[ADD0]]
; CHECK-NEXT: [[K2:%.*]] = add nuw nsw i64 [[K]], 2
-; CHECK-NEXT: [[PA2:%.*]] = getelementptr double, ptr [[A]], i64 [[K2]]
; CHECK-NEXT: [[BM2:%.*]] = mul nuw nsw i64 [[K2]], [[NB]]
; CHECK-NEXT: [[PB2:%.*]] = getelementptr double, ptr [[B]], i64 [[BM2]]
; CHECK-NEXT: [[B2:%.*]] = load double, ptr [[PB2]], align 8
; CHECK-NEXT: [[K3:%.*]] = add nuw nsw i64 [[K]], 3
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PA2]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = fmul fast <2 x double> [[TMP2]], [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
-; CHECK-NEXT: [[PROD2:%.*]] = fmul fast double [[TMP4]], [[B2]]
-; CHECK-NEXT: [[DOT:%.*]] = fadd fast double [[PROD2]], [[ADD1]]
+; CHECK-NEXT: [[PA3:%.*]] = getelementptr double, ptr [[A]], i64 [[K3]]
+; CHECK-NEXT: [[A3:%.*]] = load double, ptr [[PA3]], align 8
; CHECK-NEXT: [[BM3:%.*]] = mul nuw nsw i64 [[K3]], [[NB]]
; CHECK-NEXT: [[PB3:%.*]] = getelementptr double, ptr [[B]], i64 [[BM3]]
; CHECK-NEXT: [[B3:%.*]] = load double, ptr [[PB3]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
-; CHECK-NEXT: [[OP_RDX:%.*]] = fmul fast double [[TMP5]], [[B3]]
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PA1]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[A3]], i64 2
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP4]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast <4 x double> [[TMP5]], [[TMP1]]
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 0.000000e+00>, double [[B1]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x double> [[TMP7]], double [[B2]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> [[TMP8]], double [[B3]], i64 2
+; CHECK-NEXT: [[TMP10:%.*]] = fmul fast <4 x double> [[TMP9]], [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP10]])
+; CHECK-NEXT: [[OP_RDX:%.*]] = fadd fast double [[TMP11]], [[PROD0]]
; CHECK-NEXT: [[OP_RDX1]] = fadd fast double [[OP_RDX]], [[DOT]]
; CHECK-NEXT: [[K_NEXT]] = add nuw nsw i64 [[K]], 4
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll b/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
index 79945d8372b05..b19ace8e6abdd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
@@ -13,7 +13,6 @@ define i32 @ray_sphere(ptr nocapture noundef readonly %sph, ptr nocapture nounde
; SSE2-NEXT: entry:
; SSE2-NEXT: [[DIR:%.*]] = getelementptr inbounds [[STRUCT_RAY:%.*]], ptr [[RAY:%.*]], i64 0, i32 1
; SSE2-NEXT: [[Z:%.*]] = getelementptr inbounds [[STRUCT_RAY]], ptr [[RAY]], i64 0, i32 1, i32 2
-; SSE2-NEXT: [[TMP3:%.*]] = load double, ptr [[Z]], align 8
; SSE2-NEXT: [[TMP5:%.*]] = load double, ptr [[RAY]], align 8
; SSE2-NEXT: [[TMP6:%.*]] = load double, ptr [[SPH:%.*]], align 8
; SSE2-NEXT: [[SUB:%.*]] = fsub double [[TMP5]], [[TMP6]]
@@ -31,16 +30,17 @@ define i32 @ray_sphere(ptr nocapture noundef readonly %sph, ptr nocapture nounde
; SSE2-NEXT: [[TMP45:%.*]] = fmul <2 x double> [[TMP43]], [[TMP44]]
; SSE2-NEXT: [[TMP46:%.*]] = insertelement <2 x double> [[TMP40]], double [[SUB]], i64 1
; SSE2-NEXT: [[TMP47:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP41]], <2 x double> [[TMP46]], <2 x double> [[TMP45]])
-; SSE2-NEXT: [[TMP48:%.*]] = extractelement <2 x double> [[TMP47]], i64 0
-; SSE2-NEXT: [[TMP4:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP3]], double [[TMP3]], double [[TMP48]])
-; SSE2-NEXT: [[MUL26:%.*]] = fmul double [[TMP3]], 2.000000e+00
; SSE2-NEXT: [[Z28:%.*]] = getelementptr inbounds [[STRUCT_VEC3]], ptr [[RAY]], i64 0, i32 2
; SSE2-NEXT: [[TMP10:%.*]] = load double, ptr [[Z28]], align 8
; SSE2-NEXT: [[Z30:%.*]] = getelementptr inbounds [[STRUCT_VEC3]], ptr [[SPH]], i64 0, i32 2
; SSE2-NEXT: [[TMP11:%.*]] = load double, ptr [[Z30]], align 8
; SSE2-NEXT: [[SUB31:%.*]] = fsub double [[TMP10]], [[TMP11]]
-; SSE2-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP47]], i64 1
-; SSE2-NEXT: [[TMP12:%.*]] = tail call double @llvm.fmuladd.f64(double [[MUL26]], double [[SUB31]], double [[TMP9]])
+; SSE2-NEXT: [[TMP48:%.*]] = load double, ptr [[Z]], align 8
+; SSE2-NEXT: [[TMP49:%.*]] = insertelement <2 x double> poison, double [[TMP48]], i64 0
+; SSE2-NEXT: [[TMP50:%.*]] = shufflevector <2 x double> [[TMP49]], <2 x double> poison, <2 x i32> zeroinitializer
+; SSE2-NEXT: [[TMP51:%.*]] = fmul <2 x double> [[TMP50]], <double 1.000000e+00, double 2.000000e+00>
+; SSE2-NEXT: [[TMP52:%.*]] = insertelement <2 x double> [[TMP50]], double [[SUB31]], i64 1
+; SSE2-NEXT: [[TMP53:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP51]], <2 x double> [[TMP52]], <2 x double> [[TMP47]])
; SSE2-NEXT: [[MUL42:%.*]] = fmul double [[TMP8]], [[TMP8]]
; SSE2-NEXT: [[TMP13:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP6]], double [[TMP6]], double [[MUL42]])
; SSE2-NEXT: [[TMP14:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP11]], double [[TMP11]], double [[TMP13]])
@@ -58,8 +58,10 @@ define i32 @ray_sphere(ptr nocapture noundef readonly %sph, ptr nocapture nounde
; SSE2-NEXT: [[TMP22:%.*]] = load double, ptr [[RAD]], align 8
; SSE2-NEXT: [[NEG82:%.*]] = fneg double [[TMP22]]
; SSE2-NEXT: [[TMP23:%.*]] = tail call double @llvm.fmuladd.f64(double [[NEG82]], double [[TMP22]], double [[TMP21]])
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP53]], i64 0
; SSE2-NEXT: [[TMP24:%.*]] = fmul double [[TMP4]], -4.000000e+00
; SSE2-NEXT: [[NEG86:%.*]] = fmul double [[TMP24]], [[TMP23]]
+; SSE2-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP53]], i64 1
; SSE2-NEXT: [[TMP25:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP12]], double [[TMP12]], double [[NEG86]])
; SSE2-NEXT: [[CMP:%.*]] = fcmp olt double [[TMP25]], 0.000000e+00
; SSE2-NEXT: br i1 [[CMP]], label [[CLEANUP:%.*]], label [[IF_END:%.*]]
@@ -70,7 +72,7 @@ define i32 @ray_sphere(ptr nocapture noundef readonly %sph, ptr nocapture nounde
; SSE2-NEXT: [[TMP26:%.*]] = insertelement <2 x double> poison, double [[FNEG87]], i64 0
; SSE2-NEXT: [[TMP27:%.*]] = insertelement <2 x double> [[TMP26]], double [[CALL]], i64 1
; SSE2-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP27]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
-; SSE2-NEXT: [[TMP29:%.*]] = insertelement <2 x double> [[TMP28]], double [[TMP12]], i64 1
+; SSE2-NEXT: [[TMP29:%.*]] = shufflevector <2 x double> [[TMP28]], <2 x double> [[TMP53]], <2 x i32> <i32 0, i32 3>
; SSE2-NEXT: [[TMP30:%.*]] = fsub <2 x double> [[TMP27]], [[TMP29]]
; SSE2-NEXT: [[TMP31:%.*]] = insertelement <2 x double> poison, double [[MUL88]], i64 0
; SSE2-NEXT: [[TMP32:%.*]] = shufflevector <2 x double> [[TMP31]], <2 x double> poison, <2 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/dependent-seeds.ll b/llvm/test/Transforms/SLPVectorizer/X86/dependent-seeds.ll
index db2e8017e3c27..b97a62c3b2538 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/dependent-seeds.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/dependent-seeds.ll
@@ -34,12 +34,11 @@ define double @second_level_seeds(ptr noalias %x, ptr noalias %y, ptr noalias %m
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[X]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = fadd fast double [[TMP4]], 1.000000e+00
+; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <2 x double> [[TMP3]], splat (double 1.000000e+00)
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i64 0
; CHECK-NEXT: store double [[TMP5]], ptr [[M]], align 8
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
-; CHECK-NEXT: [[TMP6:%.*]] = fadd fast double [[TMP7]], 1.000000e+00
; CHECK-NEXT: [[L:%.*]] = load double, ptr [[M]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i64 1
; CHECK-NEXT: [[R:%.*]] = fadd double [[TMP6]], [[L]]
; CHECK-NEXT: ret double [[R]]
;
More information about the llvm-commits
mailing list