[llvm] [X86][Codegen] - Introduce a new tuning for X86 backend to control FMA generation (PR #210343)
Rohit Aggarwal via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 07:09:49 PDT 2026
https://github.com/rohitaggarwal007 created https://github.com/llvm/llvm-project/pull/210343
Introduce TuningSuppressFMAFusion in X86.td and use it in visitFADD/FSUB FMA combine. Default remains unchanged (full fusion with -ffp-contract=fast); tuning opt-in restricts fusion to contract-marked FMULs. Added acc-plus-mul codegen test.
On enabling the tuning, FMA is not generated.
### Pattern 1
- acc = acc + x * c[i]
- 33% uplift is observe on enabling the tuning
### Pattern 2 (tight chained)
- acc = acc * x + c[i]
- 33% slowdown is observed on enabling the tuning
There is a follow up PR planned to handle the regressing case.
>From ee72d984f9dd4ba493c07e557c00365b85e0cd5d Mon Sep 17 00:00:00 2001
From: Rohit Aggarwal <Rohit.Aggarwal at amd.com>
Date: Fri, 17 Jul 2026 19:23:08 +0530
Subject: [PATCH] [X86][Codegen] - Tuning in X86 backend to control FMA
generation
Introduce TuningSuppressFMAFusion in X86.td and use it in visitFADD/FSUB
FMA combine. Default remains unchanged (full fusion with -ffp-contract=fast);
tuning opt-in restricts fusion to contract-marked FMULs.
Added acc-plus-mul codegen test.
Pattern 1
- acc = acc + x * c[i]
- 33% uplift is observe on enabling the tuning
Pattern 2 (tight chained)
- acc = acc * x + c[i]
- 33% slowdown is observed on enabling the tuning
There is a follow up PR planned to handle the regressing case.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 7 +
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 14 +-
llvm/lib/Target/X86/X86.td | 8 +
llvm/lib/Target/X86/X86ISelLowering.cpp | 5 +
llvm/lib/Target/X86/X86ISelLowering.h | 3 +
.../CodeGen/X86/fma-suppress-acc-plus-mul.ll | 146 ++++++++++++++++++
llvm/test/CodeGen/X86/fma-suppress.ll | 70 +++++++++
7 files changed, 251 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/fma-suppress-acc-plus-mul.ll
create mode 100644 llvm/test/CodeGen/X86/fma-suppress.ll
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 9a6dd7735421e..ac46511d92975 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3438,6 +3438,13 @@ class LLVM_ABI TargetLoweringBase {
return false;
}
+ /// When true, FMA fusion during DAG combine requires the FMUL to carry a
+ /// local contract flag, ignoring global -ffp-contract=fast.
+ virtual bool
+ restrictFMAFusionToLocalContract(const MachineFunction &MF) const {
+ return false;
+ }
+
/// Return true if an FMA operation is faster than a pair of fmul and fadd
/// instructions. fmuladd intrinsics will be expanded to FMAs when this method
/// returns true, otherwise fmuladd is expanded to fmul + fadd.
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index aba6d8c5ac310..dfc3e5cb97364 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -18451,6 +18451,8 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
// Always prefer FMAD to FMA for precision.
unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
+ bool RestrictToLocalContract =
+ TLI.restrictFMAFusionToLocalContract(DAG.getMachineFunction());
auto isFusedOp = [&](SDValue N) {
unsigned Opcode = N.getOpcode();
@@ -18459,9 +18461,12 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
// Is the node an FMUL and contractable either due to global flags or
// SDNodeFlags.
- auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
+ auto isContractableFMUL = [AllowFusionGlobally,
+ RestrictToLocalContract](SDValue N) {
if (N.getOpcode() != ISD::FMUL)
return false;
+ if (RestrictToLocalContract)
+ return N->getFlags().hasAllowContract();
return AllowFusionGlobally || N->getFlags().hasAllowContract();
};
// If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
@@ -18677,12 +18682,17 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
bool NoSignedZero = Flags.hasNoSignedZeros();
+ bool RestrictToLocalContract =
+ TLI.restrictFMAFusionToLocalContract(DAG.getMachineFunction());
// Is the node an FMUL and contractable either due to global flags or
// SDNodeFlags.
- auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
+ auto isContractableFMUL = [AllowFusionGlobally,
+ RestrictToLocalContract](SDValue N) {
if (N.getOpcode() != ISD::FMUL)
return false;
+ if (RestrictToLocalContract)
+ return N->getFlags().hasAllowContract();
return AllowFusionGlobally || N->getFlags().hasAllowContract();
};
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index b724a12d2f698..a124428e20916 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -708,6 +708,14 @@ def TuningInsertVZEROUPPER
"true", "Should insert vzeroupper instructions",
[], InlineIgnore>;
+// When enabled, only fuse mul+add into FMA when the FMUL carries a local
+// contract flag, ignoring global -ffp-contract=fast.
+def TuningSuppressFMAFusion
+ : SubtargetFeature<"suppress-fma-fusion",
+ "SuppressFMAFusion",
+ "true",
+ "Restrict FMA fusion to FMUL ops with a local contract flag">;
+
// TuningFastScalarFSQRT should be enabled if scalar FSQRT has shorter latency
// than the corresponding NR code. TuningFastVectorFSQRT should be enabled if
// vector FSQRT has higher throughput than the corresponding NR code.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 97e783c09f0f2..ce4236a2b1d22 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -36336,6 +36336,11 @@ bool X86TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
return true;
}
+bool X86TargetLowering::restrictFMAFusionToLocalContract(
+ const MachineFunction &MF) const {
+ return Subtarget.suppressFMAFusion();
+}
+
bool X86TargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
EVT VT) const {
if (Subtarget.useSoftFloat())
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 798050028c15a..7180927af7f2b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -506,6 +506,9 @@ namespace llvm {
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
EVT VT) const override;
+ bool
+ restrictFMAFusionToLocalContract(const MachineFunction &MF) const override;
+
/// Return true if it's profitable to narrow operations of type SrcVT to
/// DestVT. e.g. on x86, it's profitable to narrow from i32 to i8 but not
/// from i32 to i16.
diff --git a/llvm/test/CodeGen/X86/fma-suppress-acc-plus-mul.ll b/llvm/test/CodeGen/X86/fma-suppress-acc-plus-mul.ll
new file mode 100644
index 0000000000000..d20848f0fa584
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fma-suppress-acc-plus-mul.ll
@@ -0,0 +1,146 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+fma -fp-contract=fast | FileCheck %s --check-prefixes=FMA
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+fma,+suppress-fma-fusion -fp-contract=fast | FileCheck %s --check-prefixes=SUPPRESS
+
+; Microbenchmark pattern from acc_plus_mul_chain (run_acc_plus_mul_bench.sh):
+; acc = acc + x * c[i]
+; Non-contract IR: addend (acc) is independent of the multiply operands, so
+; +suppress-fma-fusion keeps separate mul+add instead of vfmadd231ss.
+
+define float @acc_plus_mul_chain(ptr nocapture readonly %c, float %x, i32 %n) {
+; FMA-LABEL: acc_plus_mul_chain:
+; FMA: # %bb.0: # %entry
+; FMA-NEXT: testl %esi, %esi
+; FMA-NEXT: jle .LBB0_1
+; FMA-NEXT: # %bb.3: # %for.body.preheader
+; FMA-NEXT: movl %esi, %eax
+; FMA-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; FMA-NEXT: .p2align 4
+; FMA-NEXT: .LBB0_4: # %for.body
+; FMA-NEXT: # =>This Inner Loop Header: Depth=1
+; FMA-NEXT: vfmadd231ss {{.*#+}} xmm1 = (xmm0 * mem) + xmm1
+; FMA-NEXT: addq $4, %rdi
+; FMA-NEXT: decq %rax
+; FMA-NEXT: jne .LBB0_4
+; FMA-NEXT: # %bb.2: # %for.cond.cleanup
+; FMA-NEXT: vmovaps %xmm1, %xmm0
+; FMA-NEXT: retq
+; FMA-NEXT: .LBB0_1:
+; FMA-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; FMA-NEXT: vmovaps %xmm1, %xmm0
+; FMA-NEXT: retq
+;
+; SUPPRESS-LABEL: acc_plus_mul_chain:
+; SUPPRESS: # %bb.0: # %entry
+; SUPPRESS-NEXT: testl %esi, %esi
+; SUPPRESS-NEXT: jle .LBB0_1
+; SUPPRESS-NEXT: # %bb.3: # %for.body.preheader
+; SUPPRESS-NEXT: movl %esi, %eax
+; SUPPRESS-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; SUPPRESS-NEXT: .p2align 4
+; SUPPRESS-NEXT: .LBB0_4: # %for.body
+; SUPPRESS-NEXT: # =>This Inner Loop Header: Depth=1
+; SUPPRESS-NEXT: vmulss (%rdi), %xmm0, %xmm2
+; SUPPRESS-NEXT: vaddss %xmm2, %xmm1, %xmm1
+; SUPPRESS-NEXT: addq $4, %rdi
+; SUPPRESS-NEXT: decq %rax
+; SUPPRESS-NEXT: jne .LBB0_4
+; SUPPRESS-NEXT: # %bb.2: # %for.cond.cleanup
+; SUPPRESS-NEXT: vmovaps %xmm1, %xmm0
+; SUPPRESS-NEXT: retq
+; SUPPRESS-NEXT: .LBB0_1:
+; SUPPRESS-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; SUPPRESS-NEXT: vmovaps %xmm1, %xmm0
+; SUPPRESS-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %n, 0
+ br i1 %cmp, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %wide.trip.count = zext i32 %n to i64
+ br label %for.body
+
+for.cond.cleanup:
+ %acc.lcssa = phi float [ 0.0, %entry ], [ %add, %for.body ]
+ ret float %acc.lcssa
+
+for.body:
+ %iv = phi i64 [ 0, %for.body.preheader ], [ %iv.next, %for.body ]
+ %acc = phi float [ 0.0, %for.body.preheader ], [ %add, %for.body ]
+ %ptr = getelementptr float, ptr %c, i64 %iv
+ %coeff = load float, ptr %ptr, align 4
+ %mul = fmul float %x, %coeff
+ %add = fadd float %acc, %mul
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %wide.trip.count
+ br i1 %done, label %for.cond.cleanup, label %for.body
+}
+
+; Local contract on the multiply still fuses when suppression is on.
+define float @acc_plus_mul_chain_contract(ptr nocapture readonly %c, float %x, i32 %n) {
+; FMA-LABEL: acc_plus_mul_chain_contract:
+; FMA: # %bb.0: # %entry
+; FMA-NEXT: testl %esi, %esi
+; FMA-NEXT: jle .LBB1_1
+; FMA-NEXT: # %bb.3: # %for.body.preheader
+; FMA-NEXT: movl %esi, %eax
+; FMA-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; FMA-NEXT: .p2align 4
+; FMA-NEXT: .LBB1_4: # %for.body
+; FMA-NEXT: # =>This Inner Loop Header: Depth=1
+; FMA-NEXT: vfmadd231ss {{.*#+}} xmm1 = (xmm0 * mem) + xmm1
+; FMA-NEXT: addq $4, %rdi
+; FMA-NEXT: decq %rax
+; FMA-NEXT: jne .LBB1_4
+; FMA-NEXT: # %bb.2: # %for.cond.cleanup
+; FMA-NEXT: vmovaps %xmm1, %xmm0
+; FMA-NEXT: retq
+; FMA-NEXT: .LBB1_1:
+; FMA-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; FMA-NEXT: vmovaps %xmm1, %xmm0
+; FMA-NEXT: retq
+;
+; SUPPRESS-LABEL: acc_plus_mul_chain_contract:
+; SUPPRESS: # %bb.0: # %entry
+; SUPPRESS-NEXT: testl %esi, %esi
+; SUPPRESS-NEXT: jle .LBB1_1
+; SUPPRESS-NEXT: # %bb.3: # %for.body.preheader
+; SUPPRESS-NEXT: movl %esi, %eax
+; SUPPRESS-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; SUPPRESS-NEXT: .p2align 4
+; SUPPRESS-NEXT: .LBB1_4: # %for.body
+; SUPPRESS-NEXT: # =>This Inner Loop Header: Depth=1
+; SUPPRESS-NEXT: vfmadd231ss {{.*#+}} xmm1 = (xmm0 * mem) + xmm1
+; SUPPRESS-NEXT: addq $4, %rdi
+; SUPPRESS-NEXT: decq %rax
+; SUPPRESS-NEXT: jne .LBB1_4
+; SUPPRESS-NEXT: # %bb.2: # %for.cond.cleanup
+; SUPPRESS-NEXT: vmovaps %xmm1, %xmm0
+; SUPPRESS-NEXT: retq
+; SUPPRESS-NEXT: .LBB1_1:
+; SUPPRESS-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; SUPPRESS-NEXT: vmovaps %xmm1, %xmm0
+; SUPPRESS-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %n, 0
+ br i1 %cmp, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %wide.trip.count = zext i32 %n to i64
+ br label %for.body
+
+for.cond.cleanup:
+ %acc.lcssa = phi float [ 0.0, %entry ], [ %add, %for.body ]
+ ret float %acc.lcssa
+
+for.body:
+ %iv = phi i64 [ 0, %for.body.preheader ], [ %iv.next, %for.body ]
+ %acc = phi float [ 0.0, %for.body.preheader ], [ %add, %for.body ]
+ %ptr = getelementptr float, ptr %c, i64 %iv
+ %coeff = load float, ptr %ptr, align 4
+ %mul = fmul contract float %x, %coeff
+ %add = fadd contract float %acc, %mul
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %wide.trip.count
+ br i1 %done, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/CodeGen/X86/fma-suppress.ll b/llvm/test/CodeGen/X86/fma-suppress.ll
new file mode 100644
index 0000000000000..5c5af7c12df56
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fma-suppress.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+fma -fp-contract=fast | FileCheck %s --check-prefixes=FMA
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+fma,+suppress-fma-fusion -fp-contract=fast | FileCheck %s --check-prefixes=SUPPRESS
+
+; With -fp-contract=fast the FMUL/FADD pair is contracted into an FMA by
+; default. The +suppress-fma-fusion X86 tuning feature only allows the
+; contraction when the FMUL itself carries a local 'contract' flag.
+
+define float @fmul_fadd(float %a, float %b, float %c) {
+; FMA-LABEL: fmul_fadd:
+; FMA: # %bb.0:
+; FMA-NEXT: vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
+; FMA-NEXT: retq
+;
+; SUPPRESS-LABEL: fmul_fadd:
+; SUPPRESS: # %bb.0:
+; SUPPRESS-NEXT: vmulss %xmm1, %xmm0, %xmm0
+; SUPPRESS-NEXT: vaddss %xmm2, %xmm0, %xmm0
+; SUPPRESS-NEXT: retq
+ %m = fmul float %a, %b
+ %r = fadd float %m, %c
+ ret float %r
+}
+
+; The local 'contract' flag on the FMUL is honoured even with suppression on.
+define float @fmul_fadd_contract(float %a, float %b, float %c) {
+; FMA-LABEL: fmul_fadd_contract:
+; FMA: # %bb.0:
+; FMA-NEXT: vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
+; FMA-NEXT: retq
+;
+; SUPPRESS-LABEL: fmul_fadd_contract:
+; SUPPRESS: # %bb.0:
+; SUPPRESS-NEXT: vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
+; SUPPRESS-NEXT: retq
+ %m = fmul contract float %a, %b
+ %r = fadd contract float %m, %c
+ ret float %r
+}
+
+define float @fmul_fsub(float %a, float %b, float %c) {
+; FMA-LABEL: fmul_fsub:
+; FMA: # %bb.0:
+; FMA-NEXT: vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2
+; FMA-NEXT: retq
+;
+; SUPPRESS-LABEL: fmul_fsub:
+; SUPPRESS: # %bb.0:
+; SUPPRESS-NEXT: vmulss %xmm1, %xmm0, %xmm0
+; SUPPRESS-NEXT: vsubss %xmm2, %xmm0, %xmm0
+; SUPPRESS-NEXT: retq
+ %m = fmul float %a, %b
+ %r = fsub float %m, %c
+ ret float %r
+}
+
+define float @fmul_fsub_contract(float %a, float %b, float %c) {
+; FMA-LABEL: fmul_fsub_contract:
+; FMA: # %bb.0:
+; FMA-NEXT: vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2
+; FMA-NEXT: retq
+;
+; SUPPRESS-LABEL: fmul_fsub_contract:
+; SUPPRESS: # %bb.0:
+; SUPPRESS-NEXT: vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2
+; SUPPRESS-NEXT: retq
+ %m = fmul contract float %a, %b
+ %r = fsub contract float %m, %c
+ ret float %r
+}
More information about the llvm-commits
mailing list