[llvm] [X86] Form (FM)ADDSUB for a chain of alternating FSUB/FADD (PR #226429)
Tim Besard via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 03:42:14 PDT 2026
https://github.com/maleadt updated https://github.com/llvm/llvm-project/pull/226429
>From 1f90de8de9b3ca3f118c12948c7042e65b855186 Mon Sep 17 00:00:00 2001
From: Tim Besard <tim.besard at gmail.com>
Date: Thu, 24 Sep 2026 12:30:37 +0200
Subject: [PATCH 1/2] [X86] Add tests for chained alternating FSUB/FADD
(FM)ADDSUB (NFC)
Precommit tests for chains of multiply-add/sub shuffles, as in
consecutive complex multiply-adds, which currently don't form
(FM)ADDSUB.
---
.../X86/fmaddsub-combine-chain-fp16.ll | 27 +
llvm/test/CodeGen/X86/fmaddsub-combine.ll | 524 ++++++++++++++++++
llvm/test/CodeGen/X86/fmsubadd-combine.ll | 23 +
3 files changed, 574 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/fmaddsub-combine-chain-fp16.ll
diff --git a/llvm/test/CodeGen/X86/fmaddsub-combine-chain-fp16.ll b/llvm/test/CodeGen/X86/fmaddsub-combine-chain-fp16.ll
new file mode 100644
index 0000000000000..943987838c97a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fmaddsub-combine-chain-fp16.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512fp16 -o /dev/null
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512fp16,+avx512vl | FileCheck %s
+
+; Chains of FP16 multiply-add/subs are not combined into (FM)ADDSUB chains:
+; there is no 128-bit FMADDSUB for them without VLX.
+define <8 x half> @mul_addsub_chain_ph128(<8 x half> %A, <8 x half> %B, <8 x half> %C, <8 x half> %D, <8 x half> %E) {
+; CHECK-LABEL: mul_addsub_chain_ph128:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmulph %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: vsubph %xmm4, %xmm0, %xmm1
+; CHECK-NEXT: vaddph %xmm4, %xmm0, %xmm0
+; CHECK-NEXT: vmulph %xmm3, %xmm2, %xmm2
+; CHECK-NEXT: vsubph %xmm1, %xmm2, %xmm1
+; CHECK-NEXT: vaddph %xmm0, %xmm2, %xmm0
+; CHECK-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-NEXT: retq
+ %AB = fmul contract <8 x half> %A, %B
+ %Sub0 = fsub contract <8 x half> %AB, %E
+ %Add0 = fadd contract <8 x half> %AB, %E
+ %Inner = shufflevector <8 x half> %Sub0, <8 x half> %Add0, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ %CD = fmul contract <8 x half> %C, %D
+ %Sub1 = fsub contract <8 x half> %CD, %Inner
+ %Add1 = fadd contract <8 x half> %CD, %Inner
+ %Outer = shufflevector <8 x half> %Sub1, <8 x half> %Add1, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ ret <8 x half> %Outer
+}
diff --git a/llvm/test/CodeGen/X86/fmaddsub-combine.ll b/llvm/test/CodeGen/X86/fmaddsub-combine.ll
index 67a6c446afa42..b1237865ca8d8 100644
--- a/llvm/test/CodeGen/X86/fmaddsub-combine.ll
+++ b/llvm/test/CodeGen/X86/fmaddsub-combine.ll
@@ -622,3 +622,527 @@ define <16 x float> @mul_addsub_ps512_partial_avx(<16 x float> %C, <16 x float>
%vecinsert162 = shufflevector <16 x float> %vecinsert141, <16 x float> %i15, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
ret <16 x float> %vecinsert162
}
+
+; A chain of two multiply-add/subs, as in consecutive complex multiply-adds.
+; Simplifying the demanded lanes of the outer FSUB/FADD splits the inner
+; add/sub shuffle into its FSUB and FADD.
+define <2 x double> @mul_addsub_chain_pd128(<2 x double> %A, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E) {
+; NOFMA-LABEL: mul_addsub_chain_pd128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; NOFMA-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; NOFMA-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; NOFMA-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain_pd128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA3-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; FMA3-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA3-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_pd128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA4-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; FMA4-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA4-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA4-NEXT: retq
+ %AB = fmul contract <2 x double> %A, %B
+ %Sub0 = fsub contract <2 x double> %AB, %E
+ %Add0 = fadd contract <2 x double> %AB, %E
+ %Inner = shufflevector <2 x double> %Sub0, <2 x double> %Add0, <2 x i32> <i32 0, i32 3>
+ %CD = fmul contract <2 x double> %C, %D
+ %Sub1 = fsub contract <2 x double> %CD, %Inner
+ %Add1 = fadd contract <2 x double> %CD, %Inner
+ %Outer = shufflevector <2 x double> %Sub1, <2 x double> %Add1, <2 x i32> <i32 0, i32 3>
+ ret <2 x double> %Outer
+}
+
+; Same with commuted FADD operands and the FADD as the first shuffle operand.
+define <8 x float> @mul_addsub_chain_ps256(<8 x float> %A, <8 x float> %B, <8 x float> %C, <8 x float> %D, <8 x float> %E) {
+; NOFMA-LABEL: mul_addsub_chain_ps256:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulps %ymm1, %ymm0, %ymm0
+; NOFMA-NEXT: vsubps %ymm4, %ymm0, %ymm1
+; NOFMA-NEXT: vaddps %ymm0, %ymm4, %ymm0
+; NOFMA-NEXT: vmulps %ymm3, %ymm2, %ymm2
+; NOFMA-NEXT: vsubps %ymm1, %ymm2, %ymm1
+; NOFMA-NEXT: vaddps %ymm2, %ymm0, %ymm0
+; NOFMA-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain_ps256:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vmulps %ymm1, %ymm0, %ymm0
+; FMA3-NEXT: vsubps %ymm4, %ymm0, %ymm1
+; FMA3-NEXT: vaddps %ymm0, %ymm4, %ymm0
+; FMA3-NEXT: vmulps %ymm3, %ymm2, %ymm2
+; FMA3-NEXT: vsubps %ymm1, %ymm2, %ymm1
+; FMA3-NEXT: vaddps %ymm2, %ymm0, %ymm0
+; FMA3-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_ps256:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vmulps %ymm1, %ymm0, %ymm0
+; FMA4-NEXT: vsubps %ymm4, %ymm0, %ymm1
+; FMA4-NEXT: vaddps %ymm0, %ymm4, %ymm0
+; FMA4-NEXT: vmulps %ymm3, %ymm2, %ymm2
+; FMA4-NEXT: vsubps %ymm1, %ymm2, %ymm1
+; FMA4-NEXT: vaddps %ymm2, %ymm0, %ymm0
+; FMA4-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
+; FMA4-NEXT: retq
+ %AB = fmul contract <8 x float> %A, %B
+ %Sub0 = fsub contract <8 x float> %AB, %E
+ %Add0 = fadd contract <8 x float> %E, %AB
+ %Inner = shufflevector <8 x float> %Add0, <8 x float> %Sub0, <8 x i32> <i32 8, i32 1, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
+ %CD = fmul contract <8 x float> %C, %D
+ %Sub1 = fsub contract <8 x float> %CD, %Inner
+ %Add1 = fadd contract <8 x float> %Inner, %CD
+ %Outer = shufflevector <8 x float> %Add1, <8 x float> %Sub1, <8 x i32> <i32 8, i32 1, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
+ ret <8 x float> %Outer
+}
+
+define <8 x double> @mul_addsub_chain_pd512(<8 x double> %A, <8 x double> %B, <8 x double> %C, <8 x double> %D, <8 x double> %E) {
+; NOFMA-LABEL: mul_addsub_chain_pd512:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: pushq %rbp
+; NOFMA-NEXT: .cfi_def_cfa_offset 16
+; NOFMA-NEXT: .cfi_offset %rbp, -16
+; NOFMA-NEXT: movq %rsp, %rbp
+; NOFMA-NEXT: .cfi_def_cfa_register %rbp
+; NOFMA-NEXT: andq $-32, %rsp
+; NOFMA-NEXT: subq $32, %rsp
+; NOFMA-NEXT: vmovapd 48(%rbp), %ymm8
+; NOFMA-NEXT: vmovapd 16(%rbp), %ymm9
+; NOFMA-NEXT: vmulpd %ymm3, %ymm1, %ymm1
+; NOFMA-NEXT: vmulpd %ymm2, %ymm0, %ymm0
+; NOFMA-NEXT: vsubpd %ymm9, %ymm0, %ymm2
+; NOFMA-NEXT: vsubpd %ymm8, %ymm1, %ymm3
+; NOFMA-NEXT: vaddpd %ymm0, %ymm9, %ymm0
+; NOFMA-NEXT: vaddpd %ymm1, %ymm8, %ymm1
+; NOFMA-NEXT: vmulpd %ymm6, %ymm4, %ymm4
+; NOFMA-NEXT: vmulpd %ymm7, %ymm5, %ymm5
+; NOFMA-NEXT: vsubpd %ymm3, %ymm5, %ymm3
+; NOFMA-NEXT: vsubpd %ymm2, %ymm4, %ymm2
+; NOFMA-NEXT: vaddpd %ymm1, %ymm5, %ymm1
+; NOFMA-NEXT: vblendpd {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2],ymm1[3]
+; NOFMA-NEXT: vaddpd %ymm0, %ymm4, %ymm0
+; NOFMA-NEXT: vblendpd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2],ymm0[3]
+; NOFMA-NEXT: movq %rbp, %rsp
+; NOFMA-NEXT: popq %rbp
+; NOFMA-NEXT: .cfi_def_cfa %rsp, 8
+; NOFMA-NEXT: retq
+;
+; FMA3_256-LABEL: mul_addsub_chain_pd512:
+; FMA3_256: # %bb.0:
+; FMA3_256-NEXT: pushq %rbp
+; FMA3_256-NEXT: .cfi_def_cfa_offset 16
+; FMA3_256-NEXT: .cfi_offset %rbp, -16
+; FMA3_256-NEXT: movq %rsp, %rbp
+; FMA3_256-NEXT: .cfi_def_cfa_register %rbp
+; FMA3_256-NEXT: andq $-32, %rsp
+; FMA3_256-NEXT: subq $32, %rsp
+; FMA3_256-NEXT: vmovapd 48(%rbp), %ymm8
+; FMA3_256-NEXT: vmovapd 16(%rbp), %ymm9
+; FMA3_256-NEXT: vmulpd %ymm3, %ymm1, %ymm1
+; FMA3_256-NEXT: vmulpd %ymm2, %ymm0, %ymm0
+; FMA3_256-NEXT: vsubpd %ymm9, %ymm0, %ymm2
+; FMA3_256-NEXT: vsubpd %ymm8, %ymm1, %ymm3
+; FMA3_256-NEXT: vaddpd %ymm0, %ymm9, %ymm0
+; FMA3_256-NEXT: vaddpd %ymm1, %ymm8, %ymm1
+; FMA3_256-NEXT: vmulpd %ymm6, %ymm4, %ymm4
+; FMA3_256-NEXT: vmulpd %ymm7, %ymm5, %ymm5
+; FMA3_256-NEXT: vsubpd %ymm3, %ymm5, %ymm3
+; FMA3_256-NEXT: vsubpd %ymm2, %ymm4, %ymm2
+; FMA3_256-NEXT: vaddpd %ymm1, %ymm5, %ymm1
+; FMA3_256-NEXT: vblendpd {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2],ymm1[3]
+; FMA3_256-NEXT: vaddpd %ymm0, %ymm4, %ymm0
+; FMA3_256-NEXT: vblendpd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2],ymm0[3]
+; FMA3_256-NEXT: movq %rbp, %rsp
+; FMA3_256-NEXT: popq %rbp
+; FMA3_256-NEXT: .cfi_def_cfa %rsp, 8
+; FMA3_256-NEXT: retq
+;
+; FMA3_512-LABEL: mul_addsub_chain_pd512:
+; FMA3_512: # %bb.0:
+; FMA3_512-NEXT: vmulpd %zmm1, %zmm0, %zmm0
+; FMA3_512-NEXT: vsubpd %zmm4, %zmm0, %zmm1
+; FMA3_512-NEXT: vaddpd %zmm4, %zmm0, %zmm0
+; FMA3_512-NEXT: vmulpd %zmm3, %zmm2, %zmm2
+; FMA3_512-NEXT: vsubpd %zmm1, %zmm2, %zmm1
+; FMA3_512-NEXT: vaddpd %zmm0, %zmm2, %zmm0
+; FMA3_512-NEXT: vshufpd {{.*#+}} zmm0 = zmm1[0],zmm0[1],zmm1[2],zmm0[3],zmm1[4],zmm0[5],zmm1[6],zmm0[7]
+; FMA3_512-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_pd512:
+; FMA4: # %bb.0:
+; FMA4-NEXT: pushq %rbp
+; FMA4-NEXT: .cfi_def_cfa_offset 16
+; FMA4-NEXT: .cfi_offset %rbp, -16
+; FMA4-NEXT: movq %rsp, %rbp
+; FMA4-NEXT: .cfi_def_cfa_register %rbp
+; FMA4-NEXT: andq $-32, %rsp
+; FMA4-NEXT: subq $32, %rsp
+; FMA4-NEXT: vmovapd 48(%rbp), %ymm8
+; FMA4-NEXT: vmovapd 16(%rbp), %ymm9
+; FMA4-NEXT: vmulpd %ymm3, %ymm1, %ymm1
+; FMA4-NEXT: vmulpd %ymm2, %ymm0, %ymm0
+; FMA4-NEXT: vsubpd %ymm9, %ymm0, %ymm2
+; FMA4-NEXT: vsubpd %ymm8, %ymm1, %ymm3
+; FMA4-NEXT: vaddpd %ymm0, %ymm9, %ymm0
+; FMA4-NEXT: vaddpd %ymm1, %ymm8, %ymm1
+; FMA4-NEXT: vmulpd %ymm6, %ymm4, %ymm4
+; FMA4-NEXT: vmulpd %ymm7, %ymm5, %ymm5
+; FMA4-NEXT: vsubpd %ymm3, %ymm5, %ymm3
+; FMA4-NEXT: vsubpd %ymm2, %ymm4, %ymm2
+; FMA4-NEXT: vaddpd %ymm1, %ymm5, %ymm1
+; FMA4-NEXT: vblendpd {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2],ymm1[3]
+; FMA4-NEXT: vaddpd %ymm0, %ymm4, %ymm0
+; FMA4-NEXT: vblendpd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2],ymm0[3]
+; FMA4-NEXT: movq %rbp, %rsp
+; FMA4-NEXT: popq %rbp
+; FMA4-NEXT: .cfi_def_cfa %rsp, 8
+; FMA4-NEXT: retq
+ %AB = fmul contract <8 x double> %A, %B
+ %Sub0 = fsub contract <8 x double> %AB, %E
+ %Add0 = fadd contract <8 x double> %AB, %E
+ %Inner = shufflevector <8 x double> %Sub0, <8 x double> %Add0, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ %CD = fmul contract <8 x double> %C, %D
+ %Sub1 = fsub contract <8 x double> %CD, %Inner
+ %Add1 = fadd contract <8 x double> %CD, %Inner
+ %Outer = shufflevector <8 x double> %Sub1, <8 x double> %Add1, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ ret <8 x double> %Outer
+}
+
+; Without contract flags the chain can only become two ADDSUBs.
+define <2 x double> @mul_addsub_chain_nocontract_pd128(<2 x double> %A, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E) {
+; NOFMA-LABEL: mul_addsub_chain_nocontract_pd128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; NOFMA-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; NOFMA-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; NOFMA-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain_nocontract_pd128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA3-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; FMA3-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA3-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_nocontract_pd128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA4-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; FMA4-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA4-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA4-NEXT: retq
+ %AB = fmul <2 x double> %A, %B
+ %Sub0 = fsub <2 x double> %AB, %E
+ %Add0 = fadd <2 x double> %AB, %E
+ %Inner = shufflevector <2 x double> %Sub0, <2 x double> %Add0, <2 x i32> <i32 0, i32 3>
+ %CD = fmul <2 x double> %C, %D
+ %Sub1 = fsub <2 x double> %CD, %Inner
+ %Add1 = fadd <2 x double> %CD, %Inner
+ %Outer = shufflevector <2 x double> %Sub1, <2 x double> %Add1, <2 x i32> <i32 0, i32 3>
+ ret <2 x double> %Outer
+}
+
+; A chain of three multiply-add/subs.
+define <2 x double> @mul_addsub_chain3_pd128(<2 x double> %A, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E, <2 x double> %F, <2 x double> %G) {
+; NOFMA-LABEL: mul_addsub_chain3_pd128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; NOFMA-NEXT: vsubpd %xmm6, %xmm0, %xmm1
+; NOFMA-NEXT: vaddpd %xmm6, %xmm0, %xmm0
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; NOFMA-NEXT: vmulpd %xmm5, %xmm4, %xmm2
+; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; NOFMA-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain3_pd128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA3-NEXT: vsubpd %xmm6, %xmm0, %xmm1
+; FMA3-NEXT: vaddpd %xmm6, %xmm0, %xmm0
+; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA3-NEXT: vmulpd %xmm5, %xmm4, %xmm2
+; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA3-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain3_pd128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA4-NEXT: vsubpd %xmm6, %xmm0, %xmm1
+; FMA4-NEXT: vaddpd %xmm6, %xmm0, %xmm0
+; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA4-NEXT: vmulpd %xmm5, %xmm4, %xmm2
+; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA4-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA4-NEXT: retq
+ %AB = fmul contract <2 x double> %A, %B
+ %Sub0 = fsub contract <2 x double> %AB, %G
+ %Add0 = fadd contract <2 x double> %AB, %G
+ %Inner0 = shufflevector <2 x double> %Sub0, <2 x double> %Add0, <2 x i32> <i32 0, i32 3>
+ %CD = fmul contract <2 x double> %C, %D
+ %Sub1 = fsub contract <2 x double> %CD, %Inner0
+ %Add1 = fadd contract <2 x double> %CD, %Inner0
+ %Inner1 = shufflevector <2 x double> %Sub1, <2 x double> %Add1, <2 x i32> <i32 0, i32 3>
+ %EF = fmul contract <2 x double> %E, %F
+ %Sub2 = fsub contract <2 x double> %EF, %Inner1
+ %Add2 = fadd contract <2 x double> %EF, %Inner1
+ %Outer = shufflevector <2 x double> %Sub2, <2 x double> %Add2, <2 x i32> <i32 0, i32 3>
+ ret <2 x double> %Outer
+}
+
+; Inner SUBADD feeding an outer ADDSUB.
+define <4 x float> @mul_addsub_chain_subadd_ps128(<4 x float> %A, <4 x float> %B, <4 x float> %C, <4 x float> %D, <4 x float> %E) {
+; NOFMA-LABEL: mul_addsub_chain_subadd_ps128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulps %xmm1, %xmm0, %xmm0
+; NOFMA-NEXT: vsubps %xmm4, %xmm0, %xmm1
+; NOFMA-NEXT: vaddps %xmm4, %xmm0, %xmm0
+; NOFMA-NEXT: vmulps %xmm3, %xmm2, %xmm2
+; NOFMA-NEXT: vsubps %xmm0, %xmm2, %xmm0
+; NOFMA-NEXT: vaddps %xmm1, %xmm2, %xmm1
+; NOFMA-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain_subadd_ps128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vmulps %xmm1, %xmm0, %xmm0
+; FMA3-NEXT: vsubps %xmm4, %xmm0, %xmm1
+; FMA3-NEXT: vaddps %xmm4, %xmm0, %xmm0
+; FMA3-NEXT: vmulps %xmm3, %xmm2, %xmm2
+; FMA3-NEXT: vsubps %xmm0, %xmm2, %xmm0
+; FMA3-NEXT: vaddps %xmm1, %xmm2, %xmm1
+; FMA3-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_subadd_ps128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vmulps %xmm1, %xmm0, %xmm0
+; FMA4-NEXT: vsubps %xmm4, %xmm0, %xmm1
+; FMA4-NEXT: vaddps %xmm4, %xmm0, %xmm0
+; FMA4-NEXT: vmulps %xmm3, %xmm2, %xmm2
+; FMA4-NEXT: vsubps %xmm0, %xmm2, %xmm0
+; FMA4-NEXT: vaddps %xmm1, %xmm2, %xmm1
+; FMA4-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; FMA4-NEXT: retq
+ %AB = fmul contract <4 x float> %A, %B
+ %Sub0 = fsub contract <4 x float> %AB, %E
+ %Add0 = fadd contract <4 x float> %AB, %E
+ %Inner = shufflevector <4 x float> %Add0, <4 x float> %Sub0, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+ %CD = fmul contract <4 x float> %C, %D
+ %Sub1 = fsub contract <4 x float> %CD, %Inner
+ %Add1 = fadd contract <4 x float> %CD, %Inner
+ %Outer = shufflevector <4 x float> %Sub1, <4 x float> %Add1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+ ret <4 x float> %Outer
+}
+
+; Only the outer operations are contractable.
+define <2 x double> @mul_addsub_chain_inner_nocontract_pd128(<2 x double> %A, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E) {
+; NOFMA-LABEL: mul_addsub_chain_inner_nocontract_pd128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; NOFMA-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; NOFMA-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; NOFMA-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain_inner_nocontract_pd128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA3-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; FMA3-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA3-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_inner_nocontract_pd128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA4-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; FMA4-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA4-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA4-NEXT: retq
+ %AB = fmul <2 x double> %A, %B
+ %Sub0 = fsub <2 x double> %AB, %E
+ %Add0 = fadd <2 x double> %AB, %E
+ %Inner = shufflevector <2 x double> %Sub0, <2 x double> %Add0, <2 x i32> <i32 0, i32 3>
+ %CD = fmul contract <2 x double> %C, %D
+ %Sub1 = fsub contract <2 x double> %CD, %Inner
+ %Add1 = fadd contract <2 x double> %CD, %Inner
+ %Outer = shufflevector <2 x double> %Sub1, <2 x double> %Add1, <2 x i32> <i32 0, i32 3>
+ ret <2 x double> %Outer
+}
+
+; Negative test: the second operands are unrelated. Blending them would keep
+; them from contracting into FMAs of their own.
+define <4 x float> @mul_addsub_different_rhs_ps128(<4 x float> %A, <4 x float> %B, <4 x float> %C, <4 x float> %D, <4 x float> %E) {
+; NOFMA-LABEL: mul_addsub_different_rhs_ps128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulps %xmm2, %xmm1, %xmm1
+; NOFMA-NEXT: vmulps %xmm4, %xmm3, %xmm2
+; NOFMA-NEXT: vsubps %xmm1, %xmm0, %xmm1
+; NOFMA-NEXT: vaddps %xmm2, %xmm0, %xmm0
+; NOFMA-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3]
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_different_rhs_ps128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vfnmadd213ps {{.*#+}} xmm1 = -(xmm2 * xmm1) + xmm0
+; FMA3-NEXT: vfmadd213ps {{.*#+}} xmm3 = (xmm4 * xmm3) + xmm0
+; FMA3-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm3[1],xmm1[2],xmm3[3]
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_different_rhs_ps128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vfnmaddps {{.*#+}} xmm1 = -(xmm1 * xmm2) + xmm0
+; FMA4-NEXT: vfmaddps {{.*#+}} xmm0 = (xmm3 * xmm4) + xmm0
+; FMA4-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3]
+; FMA4-NEXT: retq
+ %X = fmul contract <4 x float> %B, %C
+ %Y = fmul contract <4 x float> %D, %E
+ %Sub = fsub contract <4 x float> %A, %X
+ %Add = fadd contract <4 x float> %A, %Y
+ %Addsub = shufflevector <4 x float> %Sub, <4 x float> %Add, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+ ret <4 x float> %Addsub
+}
+
+; Negative test: the inner FSUB has another use.
+define <2 x double> @mul_addsub_chain_multiuse_pd128(<2 x double> %A, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E, ptr %p) {
+; NOFMA-LABEL: mul_addsub_chain_multiuse_pd128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; NOFMA-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; NOFMA-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; NOFMA-NEXT: vmovapd %xmm1, (%rdi)
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; NOFMA-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain_multiuse_pd128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA3-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; FMA3-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; FMA3-NEXT: vmovapd %xmm1, (%rdi)
+; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA3-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_multiuse_pd128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA4-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; FMA4-NEXT: vaddpd %xmm4, %xmm0, %xmm0
+; FMA4-NEXT: vmovapd %xmm1, (%rdi)
+; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA4-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA4-NEXT: retq
+ %AB = fmul contract <2 x double> %A, %B
+ %Sub0 = fsub contract <2 x double> %AB, %E
+ %Add0 = fadd contract <2 x double> %AB, %E
+ store <2 x double> %Sub0, ptr %p
+ %Inner = shufflevector <2 x double> %Sub0, <2 x double> %Add0, <2 x i32> <i32 0, i32 3>
+ %CD = fmul contract <2 x double> %C, %D
+ %Sub1 = fsub contract <2 x double> %CD, %Inner
+ %Add1 = fadd contract <2 x double> %CD, %Inner
+ %Outer = shufflevector <2 x double> %Sub1, <2 x double> %Add1, <2 x i32> <i32 0, i32 3>
+ ret <2 x double> %Outer
+}
+
+; Negative test: the inner FSUB and FADD have different operands.
+define <2 x double> @mul_addsub_chain_different_inner_ops_pd128(<2 x double> %A, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E, <2 x double> %F) {
+; NOFMA-LABEL: mul_addsub_chain_different_inner_ops_pd128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; NOFMA-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; NOFMA-NEXT: vaddpd %xmm5, %xmm0, %xmm0
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; NOFMA-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain_different_inner_ops_pd128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA3-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; FMA3-NEXT: vaddpd %xmm5, %xmm0, %xmm0
+; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA3-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_different_inner_ops_pd128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA4-NEXT: vsubpd %xmm4, %xmm0, %xmm1
+; FMA4-NEXT: vaddpd %xmm5, %xmm0, %xmm0
+; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm2
+; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
+; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; FMA4-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA4-NEXT: retq
+ %AB = fmul contract <2 x double> %A, %B
+ %Sub0 = fsub contract <2 x double> %AB, %E
+ %Add0 = fadd contract <2 x double> %AB, %F
+ %Inner = shufflevector <2 x double> %Sub0, <2 x double> %Add0, <2 x i32> <i32 0, i32 3>
+ %CD = fmul contract <2 x double> %C, %D
+ %Sub1 = fsub contract <2 x double> %CD, %Inner
+ %Add1 = fadd contract <2 x double> %CD, %Inner
+ %Outer = shufflevector <2 x double> %Sub1, <2 x double> %Add1, <2 x i32> <i32 0, i32 3>
+ ret <2 x double> %Outer
+}
diff --git a/llvm/test/CodeGen/X86/fmsubadd-combine.ll b/llvm/test/CodeGen/X86/fmsubadd-combine.ll
index 03bf8d1bfa9de..ac9fff8db789e 100644
--- a/llvm/test/CodeGen/X86/fmsubadd-combine.ll
+++ b/llvm/test/CodeGen/X86/fmsubadd-combine.ll
@@ -667,3 +667,26 @@ entry:
%subadd = shufflevector <2 x double> %Add, <2 x double> %Sub, <2 x i32> <i32 0, i32 3>
ret <2 x double> %subadd
}
+
+; A chain of two multiply-sub/adds: both levels could become FMSUBADD.
+define <4 x double> @mul_subadd_chain_pd256(<4 x double> %A, <4 x double> %B, <4 x double> %C, <4 x double> %D, <4 x double> %E) {
+; CHECK-LABEL: mul_subadd_chain_pd256:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmulpd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT: vsubpd %ymm4, %ymm0, %ymm1
+; CHECK-NEXT: vaddpd %ymm4, %ymm0, %ymm0
+; CHECK-NEXT: vmulpd %ymm3, %ymm2, %ymm2
+; CHECK-NEXT: vsubpd %ymm1, %ymm2, %ymm1
+; CHECK-NEXT: vaddpd %ymm0, %ymm2, %ymm0
+; CHECK-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
+; CHECK-NEXT: retq
+ %AB = fmul contract <4 x double> %A, %B
+ %Sub0 = fsub contract <4 x double> %AB, %E
+ %Add0 = fadd contract <4 x double> %AB, %E
+ %Inner = shufflevector <4 x double> %Add0, <4 x double> %Sub0, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+ %CD = fmul contract <4 x double> %C, %D
+ %Sub1 = fsub contract <4 x double> %CD, %Inner
+ %Add1 = fadd contract <4 x double> %CD, %Inner
+ %Outer = shufflevector <4 x double> %Add1, <4 x double> %Sub1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+ ret <4 x double> %Outer
+}
>From 21316702d81c1707d9078d4b321fdb871ab144bc Mon Sep 17 00:00:00 2001
From: Tim Besard <tim.besard at gmail.com>
Date: Thu, 24 Sep 2026 13:51:48 +0200
Subject: [PATCH 2/2] [X86] Form (FM)ADDSUB for a chain of alternating
FSUB/FADD
A shuffle that takes the even lanes of an FSUB and the odd lanes of an
FADD only becomes ADDSUB or FMADDSUB if both operations have the same
operands. In a chain of such operations, e.g. consecutive complex
multiply-adds, the inner add/sub shuffle is the second operand of the
outer FSUB and FADD. Before the X86 combine sees the outer shuffle, the
generic shuffle combine's SimplifyDemandedVectorElts has already
replaced that operand with the inner FSUB in the outer FSUB, and with
the inner FADD in the outer FADD, since each only demands its own lanes.
Neither level is matched then, and the chain is lowered to separate
multiplies, subtractions, additions and a blend.
Also accept an outer FSUB and FADD whose second operands are an FSUB and
an FADD with the same first operand, used only there, whose own second
operands are the same or, recursively, such a pair again. Blending them
with the outer shuffle mask recreates the inner add/sub (or sub/add)
shuffle, so a chain of two complex multiply-adds becomes two VFMADDSUBs
with FMA, or two VADDSUBs without. Unrelated second operands are left
alone, since blending them would keep them from contracting into FMAs of
their own. This is limited to f32/f64: the combine doesn't check whether
f16/bf16 FMADDSUB/ADDSUB are available (a single f16 add/sub shuffle
already fails to select without VLX), so don't extend that to chains.
Assisted-by: Claude Code, Codex
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 91 ++++++---
llvm/test/CodeGen/X86/fmaddsub-combine.ll | 236 ++++++----------------
llvm/test/CodeGen/X86/fmsubadd-combine.ll | 32 ++-
3 files changed, 148 insertions(+), 211 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e2f3b5f3cd3d5..eb2668f4dba87 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -44362,18 +44362,42 @@ static bool isAddSubOrSubAddMask(ArrayRef<int> Mask, bool &Op0Even) {
return true;
}
+/// Returns true if \p Sub and \p Add are an FSUB and an FADD with the same
+/// first operand, only used by the caller, whose second operands are the same
+/// or, recursively, such an FSUB and FADD themselves. This is what
+/// SimplifyDemandedVectorElts leaves of a chain of add/sub shuffles: each
+/// FSUB/FADD only demands its own lanes of the inner shuffle, so it is
+/// replaced by the inner FSUB or FADD respectively.
+static bool isSplitAddSubChain(SDValue Sub, SDValue Add, unsigned Depth = 0) {
+ using namespace SDPatternMatch;
+ SDValue X, Y, Z;
+ if (!sd_match(Sub, m_OneUse(m_FSub(m_Value(X), m_Value(Y)))) ||
+ !sd_match(Add, m_OneUse(m_FAdd(m_Specific(X), m_Value(Z)))))
+ return false;
+ if (Y == Z)
+ return true;
+ if (++Depth >= SelectionDAG::MaxRecursionDepth)
+ return false;
+ return isSplitAddSubChain(Y, Z, Depth) || isSplitAddSubChain(Z, Y, Depth);
+}
+
/// Returns true iff the shuffle node \p N can be replaced with ADDSUB(SUBADD)
/// operation. If true is returned then the operands of ADDSUB(SUBADD) operation
-/// are written to the parameters \p Opnd0 and \p Opnd1.
+/// are written to the parameters \p Opnd0 and \p Opnd1. \p Opnd1 and
+/// \p Opnd1Alt are the second operands of the first and the second shuffle
+/// input; if they differ, they have to be blended with the shuffle mask
+/// \p Mask to form the second operand.
///
-/// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes
-/// so it is easier to generically match. We also insert dummy vector shuffle
-/// nodes for the operands which explicitly discard the lanes which are unused
-/// by this operation to try to flow through the rest of the combiner
-/// the fact that they're unused.
+/// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle
+/// nodes so it is easier to generically match. We also insert dummy vector
+/// shuffle nodes for the operands which explicitly discard the lanes which are
+/// unused by this operation to try to flow through the rest of the combiner the
+/// fact that they're unused.
static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
+ SDValue &Opnd1Alt, SmallVectorImpl<int> &Mask,
bool &IsSubAdd, bool &HasAllowContract) {
+ using namespace SDPatternMatch;
EVT VT = N->getValueType(0);
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -44381,7 +44405,6 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
!VT.getSimpleVT().isFloatingPoint())
return false;
- SmallVector<int, 16> Mask;
SmallVector<SDValue, 2> OpInputs;
if (!getTargetShuffleInputs(SDValue(N, 0), OpInputs, Mask, DAG) ||
OpInputs.size() != 2 || isAnyZero(Mask) ||
@@ -44401,26 +44424,30 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
if (!V1->hasOneUse() || !V2->hasOneUse())
return false;
- // Ensure that both operations have the same operands. Note that we can
+ // Ensure that both operations have the same first operand. Note that we can
// commute the FADD operands.
- SDValue LHS, RHS;
- if (V1.getOpcode() == ISD::FSUB) {
- LHS = V1->getOperand(0); RHS = V1->getOperand(1);
- if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) &&
- (V2->getOperand(0) != RHS || V2->getOperand(1) != LHS))
- return false;
- } else {
- assert(V2.getOpcode() == ISD::FSUB && "Unexpected opcode");
- LHS = V2->getOperand(0); RHS = V2->getOperand(1);
- if ((V1->getOperand(0) != LHS || V1->getOperand(1) != RHS) &&
- (V1->getOperand(0) != RHS || V1->getOperand(1) != LHS))
- return false;
- }
+ SDValue Sub = V1.getOpcode() == ISD::FSUB ? V1 : V2;
+ SDValue Add = V1.getOpcode() == ISD::FSUB ? V2 : V1;
+ SDValue LHS = Sub.getOperand(0), RHS = Sub.getOperand(1), AddRHS;
+ if (!sd_match(Add, m_FAdd(m_Specific(LHS), m_Value(AddRHS))))
+ return false;
bool Op0Even;
if (!isAddSubOrSubAddMask(Mask, Op0Even))
return false;
+ // The second operands must be the same too, unless they are what is left of
+ // an inner add/sub (or sub/add) shuffle, as in a chain of complex
+ // multiply-adds. Blending them again recreates the inner shuffle.
+ // TODO: This is limited to f32/f64 as the (FM)ADDSUB availability for other
+ // types isn't checked for.
+ if (AddRHS != RHS) {
+ MVT SVT = VT.getSimpleVT().getScalarType();
+ if ((SVT != MVT::f32 && SVT != MVT::f64) ||
+ !(isSplitAddSubChain(RHS, AddRHS) || isSplitAddSubChain(AddRHS, RHS)))
+ return false;
+ }
+
// It's a subadd if the vector in the even parity is an FADD.
IsSubAdd = Op0Even ? V1->getOpcode() == ISD::FADD
: V2->getOpcode() == ISD::FADD;
@@ -44428,7 +44455,8 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
V1->getFlags().hasAllowContract() && V2->getFlags().hasAllowContract();
Opnd0 = LHS;
- Opnd1 = RHS;
+ Opnd1 = V1 == Sub ? RHS : AddRHS;
+ Opnd1Alt = V1 == Sub ? AddRHS : RHS;
return true;
}
@@ -44481,21 +44509,30 @@ static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N, const SDLoc &DL,
if (SDValue V = combineShuffleToFMAddSub(N, DL, Subtarget, DAG))
return V;
- SDValue Opnd0, Opnd1;
+ SDValue Opnd0, Opnd1, Opnd1Alt;
+ SmallVector<int, 16> Mask;
bool IsSubAdd;
bool HasAllowContract;
- if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd,
- HasAllowContract))
+ if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, Opnd1Alt, Mask,
+ IsSubAdd, HasAllowContract))
return SDValue();
MVT VT = N->getSimpleValueType(0);
+ // Blend the second operands if the FSUB and FADD have different ones. Only
+ // do so once we know that the match is used.
+ auto BlendOpnd1 = [&](SDValue Opnd) {
+ if (Opnd == Opnd1Alt)
+ return Opnd;
+ return DAG.getVectorShuffle(VT, DL, Opnd, Opnd1Alt, Mask);
+ };
+
// Try to generate X86ISD::FMADDSUB node here.
SDValue Opnd2;
if (isFMAddSubOrFMSubAdd(Subtarget, Opnd0, Opnd1, Opnd2, 2,
HasAllowContract)) {
unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
- return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
+ return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, BlendOpnd1(Opnd2));
}
if (IsSubAdd)
@@ -44513,7 +44550,7 @@ static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N, const SDLoc &DL,
if (VT.getVectorElementType() == MVT::f16)
return SDValue();
- return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
+ return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, BlendOpnd1(Opnd1));
}
/// If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the
diff --git a/llvm/test/CodeGen/X86/fmaddsub-combine.ll b/llvm/test/CodeGen/X86/fmaddsub-combine.ll
index b1237865ca8d8..756928716f94d 100644
--- a/llvm/test/CodeGen/X86/fmaddsub-combine.ll
+++ b/llvm/test/CodeGen/X86/fmaddsub-combine.ll
@@ -630,34 +630,21 @@ define <2 x double> @mul_addsub_chain_pd128(<2 x double> %A, <2 x double> %B, <2
; NOFMA-LABEL: mul_addsub_chain_pd128:
; NOFMA: # %bb.0:
; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; NOFMA-NEXT: vsubpd %xmm4, %xmm0, %xmm1
-; NOFMA-NEXT: vaddpd %xmm4, %xmm0, %xmm0
-; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; NOFMA-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm1
+; NOFMA-NEXT: vaddsubpd %xmm4, %xmm0, %xmm0
+; NOFMA-NEXT: vaddsubpd %xmm0, %xmm1, %xmm0
; NOFMA-NEXT: retq
;
; FMA3-LABEL: mul_addsub_chain_pd128:
; FMA3: # %bb.0:
-; FMA3-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; FMA3-NEXT: vsubpd %xmm4, %xmm0, %xmm1
-; FMA3-NEXT: vaddpd %xmm4, %xmm0, %xmm0
-; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; FMA3-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA3-NEXT: vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm4
+; FMA3-NEXT: vfmaddsub231pd {{.*#+}} xmm0 = (xmm3 * xmm2) +/- xmm0
; FMA3-NEXT: retq
;
; FMA4-LABEL: mul_addsub_chain_pd128:
; FMA4: # %bb.0:
-; FMA4-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; FMA4-NEXT: vsubpd %xmm4, %xmm0, %xmm1
-; FMA4-NEXT: vaddpd %xmm4, %xmm0, %xmm0
-; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; FMA4-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} xmm0 = (xmm0 * xmm1) +/- xmm4
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} xmm0 = (xmm2 * xmm3) +/- xmm0
; FMA4-NEXT: retq
%AB = fmul contract <2 x double> %A, %B
%Sub0 = fsub contract <2 x double> %AB, %E
@@ -675,34 +662,21 @@ define <8 x float> @mul_addsub_chain_ps256(<8 x float> %A, <8 x float> %B, <8 x
; NOFMA-LABEL: mul_addsub_chain_ps256:
; NOFMA: # %bb.0:
; NOFMA-NEXT: vmulps %ymm1, %ymm0, %ymm0
-; NOFMA-NEXT: vsubps %ymm4, %ymm0, %ymm1
-; NOFMA-NEXT: vaddps %ymm0, %ymm4, %ymm0
-; NOFMA-NEXT: vmulps %ymm3, %ymm2, %ymm2
-; NOFMA-NEXT: vsubps %ymm1, %ymm2, %ymm1
-; NOFMA-NEXT: vaddps %ymm2, %ymm0, %ymm0
-; NOFMA-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
+; NOFMA-NEXT: vmulps %ymm3, %ymm2, %ymm1
+; NOFMA-NEXT: vaddsubps %ymm4, %ymm0, %ymm0
+; NOFMA-NEXT: vaddsubps %ymm0, %ymm1, %ymm0
; NOFMA-NEXT: retq
;
; FMA3-LABEL: mul_addsub_chain_ps256:
; FMA3: # %bb.0:
-; FMA3-NEXT: vmulps %ymm1, %ymm0, %ymm0
-; FMA3-NEXT: vsubps %ymm4, %ymm0, %ymm1
-; FMA3-NEXT: vaddps %ymm0, %ymm4, %ymm0
-; FMA3-NEXT: vmulps %ymm3, %ymm2, %ymm2
-; FMA3-NEXT: vsubps %ymm1, %ymm2, %ymm1
-; FMA3-NEXT: vaddps %ymm2, %ymm0, %ymm0
-; FMA3-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
+; FMA3-NEXT: vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm4
+; FMA3-NEXT: vfmaddsub231ps {{.*#+}} ymm0 = (ymm3 * ymm2) +/- ymm0
; FMA3-NEXT: retq
;
; FMA4-LABEL: mul_addsub_chain_ps256:
; FMA4: # %bb.0:
-; FMA4-NEXT: vmulps %ymm1, %ymm0, %ymm0
-; FMA4-NEXT: vsubps %ymm4, %ymm0, %ymm1
-; FMA4-NEXT: vaddps %ymm0, %ymm4, %ymm0
-; FMA4-NEXT: vmulps %ymm3, %ymm2, %ymm2
-; FMA4-NEXT: vsubps %ymm1, %ymm2, %ymm1
-; FMA4-NEXT: vaddps %ymm2, %ymm0, %ymm0
-; FMA4-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
+; FMA4-NEXT: vfmaddsubps {{.*#+}} ymm0 = (ymm0 * ymm1) +/- ymm4
+; FMA4-NEXT: vfmaddsubps {{.*#+}} ymm0 = (ymm2 * ymm3) +/- ymm0
; FMA4-NEXT: retq
%AB = fmul contract <8 x float> %A, %B
%Sub0 = fsub contract <8 x float> %AB, %E
@@ -725,22 +699,14 @@ define <8 x double> @mul_addsub_chain_pd512(<8 x double> %A, <8 x double> %B, <8
; NOFMA-NEXT: .cfi_def_cfa_register %rbp
; NOFMA-NEXT: andq $-32, %rsp
; NOFMA-NEXT: subq $32, %rsp
-; NOFMA-NEXT: vmovapd 48(%rbp), %ymm8
-; NOFMA-NEXT: vmovapd 16(%rbp), %ymm9
; NOFMA-NEXT: vmulpd %ymm3, %ymm1, %ymm1
; NOFMA-NEXT: vmulpd %ymm2, %ymm0, %ymm0
-; NOFMA-NEXT: vsubpd %ymm9, %ymm0, %ymm2
-; NOFMA-NEXT: vsubpd %ymm8, %ymm1, %ymm3
-; NOFMA-NEXT: vaddpd %ymm0, %ymm9, %ymm0
-; NOFMA-NEXT: vaddpd %ymm1, %ymm8, %ymm1
-; NOFMA-NEXT: vmulpd %ymm6, %ymm4, %ymm4
-; NOFMA-NEXT: vmulpd %ymm7, %ymm5, %ymm5
-; NOFMA-NEXT: vsubpd %ymm3, %ymm5, %ymm3
-; NOFMA-NEXT: vsubpd %ymm2, %ymm4, %ymm2
-; NOFMA-NEXT: vaddpd %ymm1, %ymm5, %ymm1
-; NOFMA-NEXT: vblendpd {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2],ymm1[3]
-; NOFMA-NEXT: vaddpd %ymm0, %ymm4, %ymm0
-; NOFMA-NEXT: vblendpd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2],ymm0[3]
+; NOFMA-NEXT: vmulpd %ymm7, %ymm5, %ymm2
+; NOFMA-NEXT: vmulpd %ymm6, %ymm4, %ymm3
+; NOFMA-NEXT: vaddsubpd 16(%rbp), %ymm0, %ymm0
+; NOFMA-NEXT: vaddsubpd %ymm0, %ymm3, %ymm0
+; NOFMA-NEXT: vaddsubpd 48(%rbp), %ymm1, %ymm1
+; NOFMA-NEXT: vaddsubpd %ymm1, %ymm2, %ymm1
; NOFMA-NEXT: movq %rbp, %rsp
; NOFMA-NEXT: popq %rbp
; NOFMA-NEXT: .cfi_def_cfa %rsp, 8
@@ -755,22 +721,10 @@ define <8 x double> @mul_addsub_chain_pd512(<8 x double> %A, <8 x double> %B, <8
; FMA3_256-NEXT: .cfi_def_cfa_register %rbp
; FMA3_256-NEXT: andq $-32, %rsp
; FMA3_256-NEXT: subq $32, %rsp
-; FMA3_256-NEXT: vmovapd 48(%rbp), %ymm8
-; FMA3_256-NEXT: vmovapd 16(%rbp), %ymm9
-; FMA3_256-NEXT: vmulpd %ymm3, %ymm1, %ymm1
-; FMA3_256-NEXT: vmulpd %ymm2, %ymm0, %ymm0
-; FMA3_256-NEXT: vsubpd %ymm9, %ymm0, %ymm2
-; FMA3_256-NEXT: vsubpd %ymm8, %ymm1, %ymm3
-; FMA3_256-NEXT: vaddpd %ymm0, %ymm9, %ymm0
-; FMA3_256-NEXT: vaddpd %ymm1, %ymm8, %ymm1
-; FMA3_256-NEXT: vmulpd %ymm6, %ymm4, %ymm4
-; FMA3_256-NEXT: vmulpd %ymm7, %ymm5, %ymm5
-; FMA3_256-NEXT: vsubpd %ymm3, %ymm5, %ymm3
-; FMA3_256-NEXT: vsubpd %ymm2, %ymm4, %ymm2
-; FMA3_256-NEXT: vaddpd %ymm1, %ymm5, %ymm1
-; FMA3_256-NEXT: vblendpd {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2],ymm1[3]
-; FMA3_256-NEXT: vaddpd %ymm0, %ymm4, %ymm0
-; FMA3_256-NEXT: vblendpd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2],ymm0[3]
+; FMA3_256-NEXT: vfmaddsub213pd {{.*#+}} ymm0 = (ymm2 * ymm0) +/- mem
+; FMA3_256-NEXT: vfmaddsub231pd {{.*#+}} ymm0 = (ymm6 * ymm4) +/- ymm0
+; FMA3_256-NEXT: vfmaddsub213pd {{.*#+}} ymm1 = (ymm3 * ymm1) +/- mem
+; FMA3_256-NEXT: vfmaddsub231pd {{.*#+}} ymm1 = (ymm7 * ymm5) +/- ymm1
; FMA3_256-NEXT: movq %rbp, %rsp
; FMA3_256-NEXT: popq %rbp
; FMA3_256-NEXT: .cfi_def_cfa %rsp, 8
@@ -778,13 +732,8 @@ define <8 x double> @mul_addsub_chain_pd512(<8 x double> %A, <8 x double> %B, <8
;
; FMA3_512-LABEL: mul_addsub_chain_pd512:
; FMA3_512: # %bb.0:
-; FMA3_512-NEXT: vmulpd %zmm1, %zmm0, %zmm0
-; FMA3_512-NEXT: vsubpd %zmm4, %zmm0, %zmm1
-; FMA3_512-NEXT: vaddpd %zmm4, %zmm0, %zmm0
-; FMA3_512-NEXT: vmulpd %zmm3, %zmm2, %zmm2
-; FMA3_512-NEXT: vsubpd %zmm1, %zmm2, %zmm1
-; FMA3_512-NEXT: vaddpd %zmm0, %zmm2, %zmm0
-; FMA3_512-NEXT: vshufpd {{.*#+}} zmm0 = zmm1[0],zmm0[1],zmm1[2],zmm0[3],zmm1[4],zmm0[5],zmm1[6],zmm0[7]
+; FMA3_512-NEXT: vfmaddsub213pd {{.*#+}} zmm0 = (zmm1 * zmm0) +/- zmm4
+; FMA3_512-NEXT: vfmaddsub231pd {{.*#+}} zmm0 = (zmm3 * zmm2) +/- zmm0
; FMA3_512-NEXT: retq
;
; FMA4-LABEL: mul_addsub_chain_pd512:
@@ -796,22 +745,10 @@ define <8 x double> @mul_addsub_chain_pd512(<8 x double> %A, <8 x double> %B, <8
; FMA4-NEXT: .cfi_def_cfa_register %rbp
; FMA4-NEXT: andq $-32, %rsp
; FMA4-NEXT: subq $32, %rsp
-; FMA4-NEXT: vmovapd 48(%rbp), %ymm8
-; FMA4-NEXT: vmovapd 16(%rbp), %ymm9
-; FMA4-NEXT: vmulpd %ymm3, %ymm1, %ymm1
-; FMA4-NEXT: vmulpd %ymm2, %ymm0, %ymm0
-; FMA4-NEXT: vsubpd %ymm9, %ymm0, %ymm2
-; FMA4-NEXT: vsubpd %ymm8, %ymm1, %ymm3
-; FMA4-NEXT: vaddpd %ymm0, %ymm9, %ymm0
-; FMA4-NEXT: vaddpd %ymm1, %ymm8, %ymm1
-; FMA4-NEXT: vmulpd %ymm6, %ymm4, %ymm4
-; FMA4-NEXT: vmulpd %ymm7, %ymm5, %ymm5
-; FMA4-NEXT: vsubpd %ymm3, %ymm5, %ymm3
-; FMA4-NEXT: vsubpd %ymm2, %ymm4, %ymm2
-; FMA4-NEXT: vaddpd %ymm1, %ymm5, %ymm1
-; FMA4-NEXT: vblendpd {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2],ymm1[3]
-; FMA4-NEXT: vaddpd %ymm0, %ymm4, %ymm0
-; FMA4-NEXT: vblendpd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2],ymm0[3]
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} ymm0 = (ymm0 * ymm2) +/- mem
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} ymm0 = (ymm4 * ymm6) +/- ymm0
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} ymm1 = (ymm1 * ymm3) +/- mem
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} ymm1 = (ymm5 * ymm7) +/- ymm1
; FMA4-NEXT: movq %rbp, %rsp
; FMA4-NEXT: popq %rbp
; FMA4-NEXT: .cfi_def_cfa %rsp, 8
@@ -832,34 +769,25 @@ define <2 x double> @mul_addsub_chain_nocontract_pd128(<2 x double> %A, <2 x dou
; NOFMA-LABEL: mul_addsub_chain_nocontract_pd128:
; NOFMA: # %bb.0:
; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; NOFMA-NEXT: vsubpd %xmm4, %xmm0, %xmm1
-; NOFMA-NEXT: vaddpd %xmm4, %xmm0, %xmm0
-; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; NOFMA-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm1
+; NOFMA-NEXT: vaddsubpd %xmm4, %xmm0, %xmm0
+; NOFMA-NEXT: vaddsubpd %xmm0, %xmm1, %xmm0
; NOFMA-NEXT: retq
;
; FMA3-LABEL: mul_addsub_chain_nocontract_pd128:
; FMA3: # %bb.0:
; FMA3-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; FMA3-NEXT: vsubpd %xmm4, %xmm0, %xmm1
-; FMA3-NEXT: vaddpd %xmm4, %xmm0, %xmm0
-; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; FMA3-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm1
+; FMA3-NEXT: vaddsubpd %xmm4, %xmm0, %xmm0
+; FMA3-NEXT: vaddsubpd %xmm0, %xmm1, %xmm0
; FMA3-NEXT: retq
;
; FMA4-LABEL: mul_addsub_chain_nocontract_pd128:
; FMA4: # %bb.0:
; FMA4-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; FMA4-NEXT: vsubpd %xmm4, %xmm0, %xmm1
-; FMA4-NEXT: vaddpd %xmm4, %xmm0, %xmm0
-; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; FMA4-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm1
+; FMA4-NEXT: vaddsubpd %xmm4, %xmm0, %xmm0
+; FMA4-NEXT: vaddsubpd %xmm0, %xmm1, %xmm0
; FMA4-NEXT: retq
%AB = fmul <2 x double> %A, %B
%Sub0 = fsub <2 x double> %AB, %E
@@ -877,43 +805,25 @@ define <2 x double> @mul_addsub_chain3_pd128(<2 x double> %A, <2 x double> %B, <
; NOFMA-LABEL: mul_addsub_chain3_pd128:
; NOFMA: # %bb.0:
; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; NOFMA-NEXT: vsubpd %xmm6, %xmm0, %xmm1
-; NOFMA-NEXT: vaddpd %xmm6, %xmm0, %xmm0
-; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm1
; NOFMA-NEXT: vmulpd %xmm5, %xmm4, %xmm2
-; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; NOFMA-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; NOFMA-NEXT: vaddsubpd %xmm6, %xmm0, %xmm0
+; NOFMA-NEXT: vaddsubpd %xmm0, %xmm1, %xmm0
+; NOFMA-NEXT: vaddsubpd %xmm0, %xmm2, %xmm0
; NOFMA-NEXT: retq
;
; FMA3-LABEL: mul_addsub_chain3_pd128:
; FMA3: # %bb.0:
-; FMA3-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; FMA3-NEXT: vsubpd %xmm6, %xmm0, %xmm1
-; FMA3-NEXT: vaddpd %xmm6, %xmm0, %xmm0
-; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; FMA3-NEXT: vmulpd %xmm5, %xmm4, %xmm2
-; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; FMA3-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA3-NEXT: vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm6
+; FMA3-NEXT: vfmaddsub231pd {{.*#+}} xmm0 = (xmm3 * xmm2) +/- xmm0
+; FMA3-NEXT: vfmaddsub231pd {{.*#+}} xmm0 = (xmm5 * xmm4) +/- xmm0
; FMA3-NEXT: retq
;
; FMA4-LABEL: mul_addsub_chain3_pd128:
; FMA4: # %bb.0:
-; FMA4-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; FMA4-NEXT: vsubpd %xmm6, %xmm0, %xmm1
-; FMA4-NEXT: vaddpd %xmm6, %xmm0, %xmm0
-; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; FMA4-NEXT: vmulpd %xmm5, %xmm4, %xmm2
-; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; FMA4-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} xmm0 = (xmm0 * xmm1) +/- xmm6
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} xmm0 = (xmm2 * xmm3) +/- xmm0
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} xmm0 = (xmm4 * xmm5) +/- xmm0
; FMA4-NEXT: retq
%AB = fmul contract <2 x double> %A, %B
%Sub0 = fsub contract <2 x double> %AB, %G
@@ -937,32 +847,21 @@ define <4 x float> @mul_addsub_chain_subadd_ps128(<4 x float> %A, <4 x float> %B
; NOFMA-NEXT: vmulps %xmm1, %xmm0, %xmm0
; NOFMA-NEXT: vsubps %xmm4, %xmm0, %xmm1
; NOFMA-NEXT: vaddps %xmm4, %xmm0, %xmm0
-; NOFMA-NEXT: vmulps %xmm3, %xmm2, %xmm2
-; NOFMA-NEXT: vsubps %xmm0, %xmm2, %xmm0
-; NOFMA-NEXT: vaddps %xmm1, %xmm2, %xmm1
; NOFMA-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; NOFMA-NEXT: vmulps %xmm3, %xmm2, %xmm1
+; NOFMA-NEXT: vaddsubps %xmm0, %xmm1, %xmm0
; NOFMA-NEXT: retq
;
; FMA3-LABEL: mul_addsub_chain_subadd_ps128:
; FMA3: # %bb.0:
-; FMA3-NEXT: vmulps %xmm1, %xmm0, %xmm0
-; FMA3-NEXT: vsubps %xmm4, %xmm0, %xmm1
-; FMA3-NEXT: vaddps %xmm4, %xmm0, %xmm0
-; FMA3-NEXT: vmulps %xmm3, %xmm2, %xmm2
-; FMA3-NEXT: vsubps %xmm0, %xmm2, %xmm0
-; FMA3-NEXT: vaddps %xmm1, %xmm2, %xmm1
-; FMA3-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; FMA3-NEXT: vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm4
+; FMA3-NEXT: vfmaddsub231ps {{.*#+}} xmm0 = (xmm3 * xmm2) +/- xmm0
; FMA3-NEXT: retq
;
; FMA4-LABEL: mul_addsub_chain_subadd_ps128:
; FMA4: # %bb.0:
-; FMA4-NEXT: vmulps %xmm1, %xmm0, %xmm0
-; FMA4-NEXT: vsubps %xmm4, %xmm0, %xmm1
-; FMA4-NEXT: vaddps %xmm4, %xmm0, %xmm0
-; FMA4-NEXT: vmulps %xmm3, %xmm2, %xmm2
-; FMA4-NEXT: vsubps %xmm0, %xmm2, %xmm0
-; FMA4-NEXT: vaddps %xmm1, %xmm2, %xmm1
-; FMA4-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; FMA4-NEXT: vfmsubaddps {{.*#+}} xmm0 = (xmm0 * xmm1) -/+ xmm4
+; FMA4-NEXT: vfmaddsubps {{.*#+}} xmm0 = (xmm2 * xmm3) +/- xmm0
; FMA4-NEXT: retq
%AB = fmul contract <4 x float> %A, %B
%Sub0 = fsub contract <4 x float> %AB, %E
@@ -980,34 +879,23 @@ define <2 x double> @mul_addsub_chain_inner_nocontract_pd128(<2 x double> %A, <2
; NOFMA-LABEL: mul_addsub_chain_inner_nocontract_pd128:
; NOFMA: # %bb.0:
; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; NOFMA-NEXT: vsubpd %xmm4, %xmm0, %xmm1
-; NOFMA-NEXT: vaddpd %xmm4, %xmm0, %xmm0
-; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; NOFMA-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; NOFMA-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; NOFMA-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm1
+; NOFMA-NEXT: vaddsubpd %xmm4, %xmm0, %xmm0
+; NOFMA-NEXT: vaddsubpd %xmm0, %xmm1, %xmm0
; NOFMA-NEXT: retq
;
; FMA3-LABEL: mul_addsub_chain_inner_nocontract_pd128:
; FMA3: # %bb.0:
; FMA3-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; FMA3-NEXT: vsubpd %xmm4, %xmm0, %xmm1
-; FMA3-NEXT: vaddpd %xmm4, %xmm0, %xmm0
-; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; FMA3-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; FMA3-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; FMA3-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA3-NEXT: vaddsubpd %xmm4, %xmm0, %xmm0
+; FMA3-NEXT: vfmaddsub231pd {{.*#+}} xmm0 = (xmm3 * xmm2) +/- xmm0
; FMA3-NEXT: retq
;
; FMA4-LABEL: mul_addsub_chain_inner_nocontract_pd128:
; FMA4: # %bb.0:
; FMA4-NEXT: vmulpd %xmm1, %xmm0, %xmm0
-; FMA4-NEXT: vsubpd %xmm4, %xmm0, %xmm1
-; FMA4-NEXT: vaddpd %xmm4, %xmm0, %xmm0
-; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm2
-; FMA4-NEXT: vsubpd %xmm1, %xmm2, %xmm1
-; FMA4-NEXT: vaddpd %xmm0, %xmm2, %xmm0
-; FMA4-NEXT: vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; FMA4-NEXT: vaddsubpd %xmm4, %xmm0, %xmm0
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} xmm0 = (xmm2 * xmm3) +/- xmm0
; FMA4-NEXT: retq
%AB = fmul <2 x double> %A, %B
%Sub0 = fsub <2 x double> %AB, %E
diff --git a/llvm/test/CodeGen/X86/fmsubadd-combine.ll b/llvm/test/CodeGen/X86/fmsubadd-combine.ll
index ac9fff8db789e..edcdd132d97b1 100644
--- a/llvm/test/CodeGen/X86/fmsubadd-combine.ll
+++ b/llvm/test/CodeGen/X86/fmsubadd-combine.ll
@@ -670,16 +670,28 @@ entry:
; A chain of two multiply-sub/adds: both levels could become FMSUBADD.
define <4 x double> @mul_subadd_chain_pd256(<4 x double> %A, <4 x double> %B, <4 x double> %C, <4 x double> %D, <4 x double> %E) {
-; CHECK-LABEL: mul_subadd_chain_pd256:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vmulpd %ymm1, %ymm0, %ymm0
-; CHECK-NEXT: vsubpd %ymm4, %ymm0, %ymm1
-; CHECK-NEXT: vaddpd %ymm4, %ymm0, %ymm0
-; CHECK-NEXT: vmulpd %ymm3, %ymm2, %ymm2
-; CHECK-NEXT: vsubpd %ymm1, %ymm2, %ymm1
-; CHECK-NEXT: vaddpd %ymm0, %ymm2, %ymm0
-; CHECK-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; CHECK-NEXT: retq
+; NOFMA-LABEL: mul_subadd_chain_pd256:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulpd %ymm1, %ymm0, %ymm0
+; NOFMA-NEXT: vsubpd %ymm4, %ymm0, %ymm1
+; NOFMA-NEXT: vaddpd %ymm4, %ymm0, %ymm0
+; NOFMA-NEXT: vmulpd %ymm3, %ymm2, %ymm2
+; NOFMA-NEXT: vsubpd %ymm1, %ymm2, %ymm1
+; NOFMA-NEXT: vaddpd %ymm0, %ymm2, %ymm0
+; NOFMA-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_subadd_chain_pd256:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm4
+; FMA3-NEXT: vfmsubadd231pd {{.*#+}} ymm0 = (ymm3 * ymm2) -/+ ymm0
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_subadd_chain_pd256:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vfmsubaddpd {{.*#+}} ymm0 = (ymm0 * ymm1) -/+ ymm4
+; FMA4-NEXT: vfmsubaddpd {{.*#+}} ymm0 = (ymm2 * ymm3) -/+ ymm0
+; FMA4-NEXT: retq
%AB = fmul contract <4 x double> %A, %B
%Sub0 = fsub contract <4 x double> %AB, %E
%Add0 = fadd contract <4 x double> %AB, %E
More information about the llvm-commits
mailing list