[llvm] [SLP] Account for fma fusion when vectorizing an ordered fadd reduction (PR #210399)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 16:57:40 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/210399
>From 101f315e579ca0278f68eb8079aa29af23423bc8 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 17 Jul 2026 19:45:50 +0200
Subject: [PATCH 1/3] [SLP] Account for lost fma fusion when vectorizing an
ordered fadd reduction
An ordered fadd reduction of contractable fmuls lowers to a chain of fmas in
scalar code. SLP vectorizes the fmul operand tree and leaves the fadds as an
ordered scalar chain, which breaks that fusion, but the ordered-reduction cost
path hardcodes ReductionCost = 0 and never accounts for it. Charge the fusion
saving per reduced fmul so the cost reflects the lost fusion.
The saving is zero on targets without a faster fma, so only targets that
actually fuse are affected.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 25 +++++++++++-
.../SLPVectorizer/X86/slp-fma-loss-ordered.ll | 40 +++++++++++++++++++
2 files changed, 63 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7e22ba3bd149c..68d75fa366bc1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30217,9 +30217,30 @@ class HorizontalReduction {
// Estimate cost.
InstructionCost ReductionCost;
if (RK == ReductionOrdering::Ordered || V.isReducedBitcastRoot() ||
- V.isReducedCmpBitcastRoot())
+ V.isReducedCmpBitcastRoot()) {
ReductionCost = 0;
- else
+ // Check for potential fma fusion as vectorization would break it.
+ if (RdxKind == RecurKind::FAdd && RdxFMF.allowContract()) {
+ constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ Type *Ty = VL.front()->getType();
+ IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty},
+ RdxFMF);
+ InstructionCost FusionSaving =
+ TTI->getArithmeticInstrCost(Instruction::FMul, Ty, CostKind) +
+ TTI->getArithmeticInstrCost(Instruction::FAdd, Ty, CostKind) -
+ TTI->getIntrinsicInstrCost(ICA, CostKind);
+ if (FusionSaving.isValid() && FusionSaving > 0)
+ for (Value *RdxVal : VL) {
+ auto *FMul = dyn_cast<Instruction>(RdxVal);
+ if (FMul && FMul->getOpcode() == Instruction::FMul &&
+ FMul->hasOneUse() &&
+ cast<FPMathOperator>(FMul)
+ ->getFastMathFlags()
+ .allowContract())
+ ReductionCost += FusionSaving;
+ }
+ }
+ } else
ReductionCost =
getReductionCost(TTI, VL, SameValuesCounter, IsCmpSelMinMax,
RdxFMF, V, DT, DL, TLI);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
new file mode 100644
index 0000000000000..2aefd208e3a7f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=slp-vectorizer -S -mcpu=corei7 -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-2 < %s | FileCheck %s --check-prefixes=NOFMA
+; RUN: opt -passes=slp-vectorizer -S -mcpu=core-avx2 -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-2 < %s | FileCheck %s --check-prefixes=FMA
+
+; On a target with fast fma changing chain of fmul into fmul is not profitable
+; when fusion of fmul -> fadd sequence in fma is possible, so the reduction
+; should stay scalar.
+
+define double @mul_fun() {
+; NOFMA-LABEL: @mul_fun(
+; NOFMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; NOFMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
+; NOFMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
+; NOFMA-NEXT: [[TMP3:%.*]] = fmul contract <4 x double> <double 7.000000e+00, double -4.300000e+01, double 2.200000e-02, double 9.500000e+00>, [[TMP2]]
+; NOFMA-NEXT: [[TMP4:%.*]] = call contract double @llvm.vector.reduce.fadd.v4f64(double [[CVT0]], <4 x double> [[TMP3]])
+; NOFMA-NEXT: ret double [[TMP4]]
+;
+; FMA-LABEL: @mul_fun(
+; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; FMA-NEXT: [[MUL0:%.*]] = fmul contract double 7.000000e+00, [[CVT0]]
+; FMA-NEXT: [[ADD0:%.*]] = fadd contract double [[MUL0]], [[CVT0]]
+; FMA-NEXT: [[MUL1:%.*]] = fmul contract double -4.300000e+01, [[CVT0]]
+; FMA-NEXT: [[ADD1:%.*]] = fadd contract double [[MUL1]], [[ADD0]]
+; FMA-NEXT: [[MUL2:%.*]] = fmul contract double 2.200000e-02, [[CVT0]]
+; FMA-NEXT: [[ADD2:%.*]] = fadd contract double [[MUL2]], [[ADD1]]
+; FMA-NEXT: [[MUL3:%.*]] = fmul contract double 9.500000e+00, [[CVT0]]
+; FMA-NEXT: [[ADD3:%.*]] = fadd contract double [[MUL3]], [[ADD2]]
+; FMA-NEXT: ret double [[ADD3]]
+;
+ %cvt0 = uitofp i16 3 to double
+ %mul0 = fmul contract double 7.000000e+00, %cvt0
+ %add0 = fadd contract double %mul0, %cvt0
+ %mul1 = fmul contract double -4.300000e+01, %cvt0
+ %add1 = fadd contract double %mul1, %add0
+ %mul2 = fmul contract double 2.200000e-02, %cvt0
+ %add2 = fadd contract double %mul2, %add1
+ %mul3 = fmul contract double 9.500000e+00, %cvt0
+ %add3 = fadd contract double %mul3, %add2
+ ret double %add3
+}
>From 8f5f1e560f643280867ea9a5f4adc881a6f528f8 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 17 Jul 2026 23:21:46 +0200
Subject: [PATCH 2/3] review comments
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 36 ++++-----
.../SLPVectorizer/X86/slp-fma-loss-ordered.ll | 80 +++++++++++++++++++
2 files changed, 98 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 68d75fa366bc1..acdfb2bc83f9d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30220,25 +30220,25 @@ class HorizontalReduction {
V.isReducedCmpBitcastRoot()) {
ReductionCost = 0;
// Check for potential fma fusion as vectorization would break it.
- if (RdxKind == RecurKind::FAdd && RdxFMF.allowContract()) {
+ if (RK == ReductionOrdering::Ordered && RdxKind == RecurKind::FAdd &&
+ RdxFMF.allowContract()) {
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
- Type *Ty = VL.front()->getType();
- IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty},
- RdxFMF);
- InstructionCost FusionSaving =
- TTI->getArithmeticInstrCost(Instruction::FMul, Ty, CostKind) +
- TTI->getArithmeticInstrCost(Instruction::FAdd, Ty, CostKind) -
- TTI->getIntrinsicInstrCost(ICA, CostKind);
- if (FusionSaving.isValid() && FusionSaving > 0)
- for (Value *RdxVal : VL) {
- auto *FMul = dyn_cast<Instruction>(RdxVal);
- if (FMul && FMul->getOpcode() == Instruction::FMul &&
- FMul->hasOneUse() &&
- cast<FPMathOperator>(FMul)
- ->getFastMathFlags()
- .allowContract())
- ReductionCost += FusionSaving;
- }
+ for (Value *RdxVal : VL) {
+ auto *FMul = dyn_cast<Instruction>(RdxVal);
+ if (!FMul || FMul->getOpcode() != Instruction::FMul ||
+ !FMul->hasOneUse())
+ continue;
+ auto *FAdd = cast<Instruction>(FMul->user_back());
+ InstructionCost FMACost = canConvertToFMA(
+ FAdd, InstructionsState(FAdd, FAdd), DT, DL, *TTI, TLI);
+ if (!FMACost.isValid())
+ continue;
+ InstructionCost FusionSaving =
+ TTI->getInstructionCost(FMul, CostKind) +
+ TTI->getInstructionCost(FAdd, CostKind) - FMACost;
+ if (FusionSaving.isValid() && FusionSaving > 0)
+ ReductionCost += FusionSaving;
+ }
}
} else
ReductionCost =
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
index 2aefd208e3a7f..642d308d20feb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
@@ -38,3 +38,83 @@ define double @mul_fun() {
%add3 = fadd contract double %mul3, %add2
ret double %add3
}
+
+; %mul1 keeps a second use, so fusion was never on the table for it, it must not
+; be penalized even on an fma-capable target.
+define double @mul_fun_multiuse(ptr %dst) {
+; NOFMA-LABEL: @mul_fun_multiuse(
+; NOFMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; NOFMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
+; NOFMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
+; NOFMA-NEXT: [[TMP3:%.*]] = fmul contract <4 x double> <double 7.000000e+00, double -4.300000e+01, double 2.200000e-02, double 9.500000e+00>, [[TMP2]]
+; NOFMA-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; NOFMA-NEXT: store double [[TMP4]], ptr [[DST:%.*]], align 8
+; NOFMA-NEXT: [[TMP5:%.*]] = call contract double @llvm.vector.reduce.fadd.v4f64(double [[CVT0]], <4 x double> [[TMP3]])
+; NOFMA-NEXT: ret double [[TMP5]]
+;
+; FMA-LABEL: @mul_fun_multiuse(
+; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; FMA-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[CVT0]], i64 0
+; FMA-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> zeroinitializer
+; FMA-NEXT: [[TMP3:%.*]] = fmul contract <2 x double> <double -4.300000e+01, double 7.000000e+00>, [[TMP2]]
+; FMA-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
+; FMA-NEXT: [[ADD0:%.*]] = fadd contract double [[TMP4]], [[CVT0]]
+; FMA-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
+; FMA-NEXT: store double [[TMP5]], ptr [[DST:%.*]], align 8
+; FMA-NEXT: [[ADD1:%.*]] = fadd contract double [[TMP5]], [[ADD0]]
+; FMA-NEXT: [[MUL2:%.*]] = fmul contract double 2.200000e-02, [[CVT0]]
+; FMA-NEXT: [[ADD2:%.*]] = fadd contract double [[MUL2]], [[ADD1]]
+; FMA-NEXT: [[MUL3:%.*]] = fmul contract double 9.500000e+00, [[CVT0]]
+; FMA-NEXT: [[ADD3:%.*]] = fadd contract double [[MUL3]], [[ADD2]]
+; FMA-NEXT: ret double [[ADD3]]
+;
+ %cvt0 = uitofp i16 3 to double
+ %mul0 = fmul contract double 7.000000e+00, %cvt0
+ %add0 = fadd contract double %mul0, %cvt0
+ %mul1 = fmul contract double -4.300000e+01, %cvt0
+ store double %mul1, ptr %dst
+ %add1 = fadd contract double %mul1, %add0
+ %mul2 = fmul contract double 2.200000e-02, %cvt0
+ %add2 = fadd contract double %mul2, %add1
+ %mul3 = fmul contract double 9.500000e+00, %cvt0
+ %add3 = fadd contract double %mul3, %add2
+ ret double %add3
+}
+
+; No contract flag anywhere, so fusion was never possible. Check that vectorization
+; has happened.
+define double @mul_fun_no_contract() {
+; NOFMA-LABEL: @mul_fun_no_contract(
+; NOFMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; NOFMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
+; NOFMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
+; NOFMA-NEXT: [[TMP3:%.*]] = fmul <4 x double> <double 7.000000e+00, double -4.300000e+01, double 2.200000e-02, double 9.500000e+00>, [[TMP2]]
+; NOFMA-NEXT: [[TMP4:%.*]] = call double @llvm.vector.reduce.fadd.v4f64(double [[CVT0]], <4 x double> [[TMP3]])
+; NOFMA-NEXT: ret double [[TMP4]]
+;
+; FMA-LABEL: @mul_fun_no_contract(
+; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; FMA-NEXT: [[MUL0:%.*]] = fmul double 7.000000e+00, [[CVT0]]
+; FMA-NEXT: [[ADD0:%.*]] = fadd double [[MUL0]], [[CVT0]]
+; FMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
+; FMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
+; FMA-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[TMP2]], <double -4.300000e+01, double 2.200000e-02, double 9.500000e+00, double 1.000000e+00>
+; FMA-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i64 0
+; FMA-NEXT: [[ADD1:%.*]] = fadd double [[TMP4]], [[ADD0]]
+; FMA-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; FMA-NEXT: [[ADD2:%.*]] = fadd double [[TMP5]], [[ADD1]]
+; FMA-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[TMP3]], i64 2
+; FMA-NEXT: [[ADD3:%.*]] = fadd double [[TMP6]], [[ADD2]]
+; FMA-NEXT: ret double [[ADD3]]
+;
+ %cvt0 = uitofp i16 3 to double
+ %mul0 = fmul double 7.000000e+00, %cvt0
+ %add0 = fadd double %mul0, %cvt0
+ %mul1 = fmul double -4.300000e+01, %cvt0
+ %add1 = fadd double %mul1, %add0
+ %mul2 = fmul double 2.200000e-02, %cvt0
+ %add2 = fadd double %mul2, %add1
+ %mul3 = fmul double 9.500000e+00, %cvt0
+ %add3 = fadd double %mul3, %add2
+ ret double %add3
+}
>From e878bb5dccf2d41710c3e25fd00b99ad5ccabd20 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 18 Jul 2026 01:57:21 +0200
Subject: [PATCH 3/3] style
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c04ee0402dd8a..7ef232596b10f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30544,10 +30544,11 @@ class HorizontalReduction {
ReductionCost += FusionSaving;
}
}
- } else
+ } else {
ReductionCost =
getReductionCost(TTI, VL, SameValuesCounter, IsCmpSelMinMax,
RdxFMF, V, DT, DL, TLI);
+ }
// If the root is a select (min/max idiom), the insert point is the
// compare condition of that select.
Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
More information about the llvm-commits
mailing list