[llvm] bdedc49 - [SLP] Account for fma fusion when vectorizing an ordered fadd reduction (#210399)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 15:11:02 PDT 2026
Author: Dmitry Sidorov
Date: 2026-08-15T00:10:57+02:00
New Revision: bdedc49db2e53257b1c20aa410404ffe1674b80b
URL: https://github.com/llvm/llvm-project/commit/bdedc49db2e53257b1c20aa410404ffe1674b80b
DIFF: https://github.com/llvm/llvm-project/commit/bdedc49db2e53257b1c20aa410404ffe1674b80b.diff
LOG: [SLP] Account for fma fusion when vectorizing an ordered fadd reduction (#210399)
An ordered fadd reduction of contractable fmuls lowers to a chain of
fmas in scalar code. SLP vectorizes the fmul operand tree and leaves the
fadds as an ordered scalar chain, which breaks that fusion, but the
ordered-reduction cost path hardcodes ReductionCost = 0 and never
accounts for it. Change the fusion saving per reduced fmul so the cost
reflects the lost fusion.
The saving is zero on targets without a faster fma, so only targets that
actually fuse are affected.
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 53816d49de722..3e082dffac804 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -31109,12 +31109,41 @@ class HorizontalReduction {
// Estimate cost.
InstructionCost ReductionCost;
if (RK == ReductionOrdering::Ordered || V.isReducedBitcastRoot() ||
- V.isReducedCmpBitcastRoot())
+ V.isReducedCmpBitcastRoot()) {
ReductionCost = 0;
- else
+ // Check for potential fma fusion as vectorization would break it.
+ if (RK == ReductionOrdering::Ordered && RdxKind == RecurKind::FAdd &&
+ RdxFMF.allowContract()) {
+ constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ Type *Ty = VL.front()->getType();
+ InstructionCost UnfusedCost =
+ TTI->getArithmeticInstrCost(Instruction::FMul, Ty, CostKind) +
+ TTI->getArithmeticInstrCost(Instruction::FAdd, Ty, CostKind);
+ for (Value *RdxVal : VL) {
+ auto *FMul = dyn_cast<Instruction>(RdxVal);
+ if (!FMul || FMul->getOpcode() != Instruction::FMul ||
+ !FMul->hasOneUse())
+ continue;
+ auto *FAdd = dyn_cast<Instruction>(FMul->user_back());
+ if (!FAdd)
+ continue;
+ InstructionsState FAddS(FAdd, FAdd);
+ if (!FAddS.isAddSubLikeOp())
+ continue;
+ InstructionCost FMACost =
+ canConvertToFMA(FAdd, FAddS, DT, DL, *TTI, TLI);
+ if (!FMACost.isValid())
+ continue;
+ InstructionCost FusionSaving = UnfusedCost - FMACost;
+ if (FusionSaving.isValid() && FusionSaving > 0)
+ ReductionCost += FusionSaving;
+ }
+ }
+ } else {
ReductionCost =
getReductionCost(TTI, VL, SameValuesCounter, IsCmpSelMinMax,
RdxFMF, V, DT, DL, TLI);
+ }
// If the root is a select (min/max idiom), the insert point is the
// compare condition of that select.
Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
index 6a7a93a44f8c9..4574ff3f578bb 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
@@ -43,11 +43,11 @@ define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <16 x float> [[TMP18]], <16 x float> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
; CHECK-NEXT: [[TMP21:%.*]] = fmul contract <16 x float> [[TMP20]], [[TMP6]]
-; CHECK-NEXT: [[ACC14:%.*]] = extractelement <16 x float> [[TMP21]], i64 0
-; CHECK-NEXT: [[PROD15:%.*]] = extractelement <16 x float> [[TMP21]], i64 1
-; CHECK-NEXT: [[ACC15:%.*]] = fadd contract float [[ACC14]], [[PROD15]]
+; CHECK-NEXT: [[ACC25:%.*]] = extractelement <16 x float> [[TMP21]], i64 0
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <16 x float> [[TMP21]], i64 1
+; CHECK-NEXT: [[ACC26:%.*]] = fadd contract float [[ACC25]], [[TMP37]]
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP21]], i64 2
-; CHECK-NEXT: [[ACC16:%.*]] = fadd contract float [[ACC15]], [[TMP9]]
+; CHECK-NEXT: [[ACC16:%.*]] = fadd contract float [[ACC26]], [[TMP9]]
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x float> [[TMP21]], i64 3
; CHECK-NEXT: [[ACC17:%.*]] = fadd contract float [[ACC16]], [[TMP10]]
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <16 x float> [[TMP21]], i64 4
@@ -71,9 +71,9 @@ define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-NEXT: [[TMP35:%.*]] = extractelement <16 x float> [[TMP21]], i64 13
; CHECK-NEXT: [[ACC13:%.*]] = fadd contract float [[ACC12]], [[TMP35]]
; CHECK-NEXT: [[TMP36:%.*]] = extractelement <16 x float> [[TMP21]], i64 14
-; CHECK-NEXT: [[ACC25:%.*]] = fadd contract float [[ACC13]], [[TMP36]]
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <16 x float> [[TMP21]], i64 15
-; CHECK-NEXT: [[ACC26:%.*]] = fadd contract float [[ACC25]], [[TMP37]]
+; CHECK-NEXT: [[ACC14:%.*]] = fadd contract float [[ACC13]], [[TMP36]]
+; CHECK-NEXT: [[TMP53:%.*]] = extractelement <16 x float> [[TMP21]], i64 15
+; CHECK-NEXT: [[ACC15:%.*]] = fadd contract float [[ACC14]], [[TMP53]]
; CHECK-NEXT: [[IP16:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 25
; CHECK-NEXT: [[MP16:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 16
; CHECK-NEXT: [[IP20:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 32
@@ -85,7 +85,7 @@ define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <4 x float> [[TMP38]], <4 x float> [[TMP39]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP44:%.*]] = fmul contract <8 x float> [[TMP43]], [[TMP40]]
; CHECK-NEXT: [[TMP45:%.*]] = extractelement <8 x float> [[TMP44]], i64 0
-; CHECK-NEXT: [[ACC27:%.*]] = fadd contract float [[ACC26]], [[TMP45]]
+; CHECK-NEXT: [[ACC27:%.*]] = fadd contract float [[ACC15]], [[TMP45]]
; CHECK-NEXT: [[TMP46:%.*]] = extractelement <8 x float> [[TMP44]], i64 1
; CHECK-NEXT: [[ACC28:%.*]] = fadd contract float [[ACC27]], [[TMP46]]
; CHECK-NEXT: [[TMP47:%.*]] = extractelement <8 x float> [[TMP44]], i64 2
diff --git a/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
index a553fe126ee17..bad672c7743f6 100644
--- a/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
@@ -9,10 +9,14 @@
define float @dot_contract(float %x) {
; CHECK-LABEL: @dot_contract(
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> <float 7.000000e+00, float 3.000000e+00, float 5.000000e+00, float 9.000000e+00>, [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = call contract float @llvm.vector.reduce.fadd.v4f32(float [[X]], <4 x float> [[TMP3]])
+; CHECK-NEXT: [[M0:%.*]] = fmul contract float 7.000000e+00, [[X:%.*]]
+; CHECK-NEXT: [[A0:%.*]] = fadd contract float [[M0]], [[X]]
+; CHECK-NEXT: [[M1:%.*]] = fmul contract float 3.000000e+00, [[X]]
+; CHECK-NEXT: [[A1:%.*]] = fadd contract float [[M1]], [[A0]]
+; CHECK-NEXT: [[M2:%.*]] = fmul contract float 5.000000e+00, [[X]]
+; CHECK-NEXT: [[A2:%.*]] = fadd contract float [[M2]], [[A1]]
+; CHECK-NEXT: [[M3:%.*]] = fmul contract float 9.000000e+00, [[X]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract float [[M3]], [[A2]]
; CHECK-NEXT: ret float [[TMP4]]
;
%m0 = fmul contract float 7.000000e+00, %x
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
index 01e953d7c250c..642d308d20feb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
@@ -19,14 +19,11 @@ define double @mul_fun() {
; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
; FMA-NEXT: [[MUL0:%.*]] = fmul contract double 7.000000e+00, [[CVT0]]
; FMA-NEXT: [[ADD0:%.*]] = fadd contract double [[MUL0]], [[CVT0]]
-; FMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
-; FMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
-; FMA-NEXT: [[TMP3:%.*]] = fmul contract <4 x double> [[TMP2]], <double -4.300000e+01, double 2.200000e-02, double 9.500000e+00, double 1.000000e+00>
-; FMA-NEXT: [[MUL1:%.*]] = extractelement <4 x double> [[TMP3]], i64 0
+; FMA-NEXT: [[MUL1:%.*]] = fmul contract double -4.300000e+01, [[CVT0]]
; FMA-NEXT: [[ADD1:%.*]] = fadd contract double [[MUL1]], [[ADD0]]
-; FMA-NEXT: [[MUL2:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; FMA-NEXT: [[MUL2:%.*]] = fmul contract double 2.200000e-02, [[CVT0]]
; FMA-NEXT: [[ADD2:%.*]] = fadd contract double [[MUL2]], [[ADD1]]
-; FMA-NEXT: [[MUL3:%.*]] = extractelement <4 x double> [[TMP3]], i64 2
+; FMA-NEXT: [[MUL3:%.*]] = fmul contract double 9.500000e+00, [[CVT0]]
; FMA-NEXT: [[ADD3:%.*]] = fadd contract double [[MUL3]], [[ADD2]]
; FMA-NEXT: ret double [[ADD3]]
;
@@ -57,17 +54,17 @@ define double @mul_fun_multiuse(ptr %dst) {
;
; FMA-LABEL: @mul_fun_multiuse(
; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
-; FMA-NEXT: [[TMP4:%.*]] = fmul contract double 7.000000e+00, [[CVT0]]
+; FMA-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[CVT0]], i64 0
+; FMA-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> zeroinitializer
+; FMA-NEXT: [[TMP3:%.*]] = fmul contract <2 x double> <double -4.300000e+01, double 7.000000e+00>, [[TMP2]]
+; FMA-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
; FMA-NEXT: [[ADD0:%.*]] = fadd contract double [[TMP4]], [[CVT0]]
-; FMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
-; FMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
-; FMA-NEXT: [[TMP3:%.*]] = fmul contract <4 x double> [[TMP2]], <double -4.300000e+01, double 2.200000e-02, double 9.500000e+00, double 1.000000e+00>
-; FMA-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i64 0
+; FMA-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
; FMA-NEXT: store double [[TMP5]], ptr [[DST:%.*]], align 8
; FMA-NEXT: [[ADD1:%.*]] = fadd contract double [[TMP5]], [[ADD0]]
-; FMA-NEXT: [[MUL2:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; FMA-NEXT: [[MUL2:%.*]] = fmul contract double 2.200000e-02, [[CVT0]]
; FMA-NEXT: [[ADD2:%.*]] = fadd contract double [[MUL2]], [[ADD1]]
-; FMA-NEXT: [[MUL3:%.*]] = extractelement <4 x double> [[TMP3]], i64 2
+; FMA-NEXT: [[MUL3:%.*]] = fmul contract double 9.500000e+00, [[CVT0]]
; FMA-NEXT: [[ADD3:%.*]] = fadd contract double [[MUL3]], [[ADD2]]
; FMA-NEXT: ret double [[ADD3]]
;
More information about the llvm-commits
mailing list