[llvm] [X86] Form (FM)ADDSUB for a chain of alternating FSUB/FADD (PR #226429)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 03:32:09 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: Tim Besard (maleadt)
<details>
<summary>Changes</summary>
A shuffle selecting even FSUB lanes and odd FADD lanes becomes ADDSUB or FMADDSUB only if both operations share operands. In chains, such as SLP's consecutive complex multiply-adds, the inner add/sub shuffle is the outer pair's second operand. Before the X86 combine, SimplifyDemandedVectorElts replaces it with the inner FSUB for the outer FSUB and the inner FADD for the outer FADD. Neither level then matches, leaving separate multiplies, subtractions, additions and a blend.
Also accept second operands that are single-use FSUB/FADD pairs with a shared first operand and identical second operands, or recursively another such pair. Blending them with the outer mask recreates the inner add/sub or sub/add shuffle, producing two VFMADDSUBs with FMA or two VADDSUBs without. Leave unrelated second operands alone so they can contract into their own FMAs. Limit this to f32/f64: the combine does not check f16/bf16 FMADDSUB/ADDSUB availability, and even a single f16 shuffle fails to select without VLX.
```llvm
%AB = fmul contract <2 x double> %A, %B
%Sub0 = fsub contract <2 x double> %AB, %E
%Add0 = fadd contract <2 x double> %AB, %E
%Inner = shufflevector <2 x double> %Sub0, <2 x double> %Add0, <2 x i32> <i32 0, i32 3>
%CD = fmul contract <2 x double> %C, %D
%Sub1 = fsub contract <2 x double> %CD, %Inner
%Add1 = fadd contract <2 x double> %CD, %Inner
%Outer = shufflevector <2 x double> %Sub1, <2 x double> %Add1, <2 x i32> <i32 0, i32 3>
```
With `-mattr=+fma`: `vmulpd, vsubpd, vaddpd, vmulpd, vsubpd, vaddpd, vmovsd` becomes `vfmaddsub213pd, vfmaddsub231pd`.
Assisted by: Codex 5.5, Astra 6
Part of a series of patches developed while migrating Julia to LLVM 23
---
Patch is 28.90 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/226429.diff
4 Files Affected:
- (modified) llvm/lib/Target/X86/X86ISelLowering.cpp (+59-22)
- (added) llvm/test/CodeGen/X86/fmaddsub-combine-chain-fp16.ll (+27)
- (modified) llvm/test/CodeGen/X86/fmaddsub-combine.ll (+412)
- (modified) llvm/test/CodeGen/X86/fmsubadd-combine.ll (+35)
``````````diff
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e2f3b5f3cd3d5..e6deb82e99632 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -44362,9 +44362,31 @@ static bool isAddSubOrSubAddMask(ArrayRef<int> Mask, bool &Op0Even) {
return true;
}
+/// Returns true if \p Sub and \p Add are an FSUB and an FADD with the same
+/// first operand, only used by the caller, whose second operands are the same
+/// or, recursively, such an FSUB and FADD themselves. This is what
+/// SimplifyDemandedVectorElts leaves of a chain of add/sub shuffles: each
+/// FSUB/FADD only demands its own lanes of the inner shuffle, so it is
+/// replaced by the inner FSUB or FADD respectively.
+static bool isSplitAddSubChain(SDValue Sub, SDValue Add, unsigned Depth = 0) {
+ using namespace SDPatternMatch;
+ SDValue X, Y, Z;
+ if (!sd_match(Sub, m_OneUse(m_FSub(m_Value(X), m_Value(Y)))) ||
+ !sd_match(Add, m_OneUse(m_FAdd(m_Specific(X), m_Value(Z)))))
+ return false;
+ if (Y == Z)
+ return true;
+ if (++Depth >= SelectionDAG::MaxRecursionDepth)
+ return false;
+ return isSplitAddSubChain(Y, Z, Depth) || isSplitAddSubChain(Z, Y, Depth);
+}
+
/// Returns true iff the shuffle node \p N can be replaced with ADDSUB(SUBADD)
/// operation. If true is returned then the operands of ADDSUB(SUBADD) operation
-/// are written to the parameters \p Opnd0 and \p Opnd1.
+/// are written to the parameters \p Opnd0 and \p Opnd1. \p Opnd1 and
+/// \p Opnd1Alt are the second operands of the first and the second shuffle
+/// input; if they differ, they have to be blended with the shuffle mask
+/// \p Mask to form the second operand.
///
/// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes
/// so it is easier to generically match. We also insert dummy vector shuffle
@@ -44373,7 +44395,9 @@ static bool isAddSubOrSubAddMask(ArrayRef<int> Mask, bool &Op0Even) {
/// the fact that they're unused.
static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
+ SDValue &Opnd1Alt, SmallVectorImpl<int> &Mask,
bool &IsSubAdd, bool &HasAllowContract) {
+ using namespace SDPatternMatch;
EVT VT = N->getValueType(0);
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -44381,7 +44405,6 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
!VT.getSimpleVT().isFloatingPoint())
return false;
- SmallVector<int, 16> Mask;
SmallVector<SDValue, 2> OpInputs;
if (!getTargetShuffleInputs(SDValue(N, 0), OpInputs, Mask, DAG) ||
OpInputs.size() != 2 || isAnyZero(Mask) ||
@@ -44401,26 +44424,30 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
if (!V1->hasOneUse() || !V2->hasOneUse())
return false;
- // Ensure that both operations have the same operands. Note that we can
+ // Ensure that both operations have the same first operand. Note that we can
// commute the FADD operands.
- SDValue LHS, RHS;
- if (V1.getOpcode() == ISD::FSUB) {
- LHS = V1->getOperand(0); RHS = V1->getOperand(1);
- if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) &&
- (V2->getOperand(0) != RHS || V2->getOperand(1) != LHS))
- return false;
- } else {
- assert(V2.getOpcode() == ISD::FSUB && "Unexpected opcode");
- LHS = V2->getOperand(0); RHS = V2->getOperand(1);
- if ((V1->getOperand(0) != LHS || V1->getOperand(1) != RHS) &&
- (V1->getOperand(0) != RHS || V1->getOperand(1) != LHS))
- return false;
- }
+ SDValue Sub = V1.getOpcode() == ISD::FSUB ? V1 : V2;
+ SDValue Add = V1.getOpcode() == ISD::FSUB ? V2 : V1;
+ SDValue LHS = Sub.getOperand(0), RHS = Sub.getOperand(1), AddRHS;
+ if (!sd_match(Add, m_FAdd(m_Specific(LHS), m_Value(AddRHS))))
+ return false;
bool Op0Even;
if (!isAddSubOrSubAddMask(Mask, Op0Even))
return false;
+ // The second operands must be the same too, unless they are what is left of
+ // an inner add/sub (or sub/add) shuffle, as in a chain of complex
+ // multiply-adds. Blending them again recreates the inner shuffle.
+ // TODO: This is limited to f32/f64 as the (FM)ADDSUB availability for other
+ // types isn't checked for.
+ if (AddRHS != RHS) {
+ MVT SVT = VT.getSimpleVT().getScalarType();
+ if ((SVT != MVT::f32 && SVT != MVT::f64) ||
+ !(isSplitAddSubChain(RHS, AddRHS) || isSplitAddSubChain(AddRHS, RHS)))
+ return false;
+ }
+
// It's a subadd if the vector in the even parity is an FADD.
IsSubAdd = Op0Even ? V1->getOpcode() == ISD::FADD
: V2->getOpcode() == ISD::FADD;
@@ -44428,7 +44455,8 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
V1->getFlags().hasAllowContract() && V2->getFlags().hasAllowContract();
Opnd0 = LHS;
- Opnd1 = RHS;
+ Opnd1 = V1 == Sub ? RHS : AddRHS;
+ Opnd1Alt = V1 == Sub ? AddRHS : RHS;
return true;
}
@@ -44481,21 +44509,30 @@ static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N, const SDLoc &DL,
if (SDValue V = combineShuffleToFMAddSub(N, DL, Subtarget, DAG))
return V;
- SDValue Opnd0, Opnd1;
+ SDValue Opnd0, Opnd1, Opnd1Alt;
+ SmallVector<int, 16> Mask;
bool IsSubAdd;
bool HasAllowContract;
- if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd,
- HasAllowContract))
+ if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, Opnd1Alt, Mask,
+ IsSubAdd, HasAllowContract))
return SDValue();
MVT VT = N->getSimpleValueType(0);
+ // Blend the second operands if the FSUB and FADD have different ones. Only
+ // do so once we know that the match is used.
+ auto BlendOpnd1 = [&](SDValue Opnd) {
+ if (Opnd == Opnd1Alt)
+ return Opnd;
+ return DAG.getVectorShuffle(VT, DL, Opnd, Opnd1Alt, Mask);
+ };
+
// Try to generate X86ISD::FMADDSUB node here.
SDValue Opnd2;
if (isFMAddSubOrFMSubAdd(Subtarget, Opnd0, Opnd1, Opnd2, 2,
HasAllowContract)) {
unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
- return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
+ return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, BlendOpnd1(Opnd2));
}
if (IsSubAdd)
@@ -44513,7 +44550,7 @@ static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N, const SDLoc &DL,
if (VT.getVectorElementType() == MVT::f16)
return SDValue();
- return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
+ return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, BlendOpnd1(Opnd1));
}
/// If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the
diff --git a/llvm/test/CodeGen/X86/fmaddsub-combine-chain-fp16.ll b/llvm/test/CodeGen/X86/fmaddsub-combine-chain-fp16.ll
new file mode 100644
index 0000000000000..943987838c97a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fmaddsub-combine-chain-fp16.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512fp16 -o /dev/null
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512fp16,+avx512vl | FileCheck %s
+
+; Chains of FP16 multiply-add/subs are not combined into (FM)ADDSUB chains:
+; there is no 128-bit FMADDSUB for them without VLX.
+define <8 x half> @mul_addsub_chain_ph128(<8 x half> %A, <8 x half> %B, <8 x half> %C, <8 x half> %D, <8 x half> %E) {
+; CHECK-LABEL: mul_addsub_chain_ph128:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmulph %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: vsubph %xmm4, %xmm0, %xmm1
+; CHECK-NEXT: vaddph %xmm4, %xmm0, %xmm0
+; CHECK-NEXT: vmulph %xmm3, %xmm2, %xmm2
+; CHECK-NEXT: vsubph %xmm1, %xmm2, %xmm1
+; CHECK-NEXT: vaddph %xmm0, %xmm2, %xmm0
+; CHECK-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-NEXT: retq
+ %AB = fmul contract <8 x half> %A, %B
+ %Sub0 = fsub contract <8 x half> %AB, %E
+ %Add0 = fadd contract <8 x half> %AB, %E
+ %Inner = shufflevector <8 x half> %Sub0, <8 x half> %Add0, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ %CD = fmul contract <8 x half> %C, %D
+ %Sub1 = fsub contract <8 x half> %CD, %Inner
+ %Add1 = fadd contract <8 x half> %CD, %Inner
+ %Outer = shufflevector <8 x half> %Sub1, <8 x half> %Add1, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ ret <8 x half> %Outer
+}
diff --git a/llvm/test/CodeGen/X86/fmaddsub-combine.ll b/llvm/test/CodeGen/X86/fmaddsub-combine.ll
index 67a6c446afa42..756928716f94d 100644
--- a/llvm/test/CodeGen/X86/fmaddsub-combine.ll
+++ b/llvm/test/CodeGen/X86/fmaddsub-combine.ll
@@ -622,3 +622,415 @@ define <16 x float> @mul_addsub_ps512_partial_avx(<16 x float> %C, <16 x float>
%vecinsert162 = shufflevector <16 x float> %vecinsert141, <16 x float> %i15, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
ret <16 x float> %vecinsert162
}
+
+; A chain of two multiply-add/subs, as in consecutive complex multiply-adds.
+; Simplifying the demanded lanes of the outer FSUB/FADD splits the inner
+; add/sub shuffle into its FSUB and FADD.
+define <2 x double> @mul_addsub_chain_pd128(<2 x double> %A, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E) {
+; NOFMA-LABEL: mul_addsub_chain_pd128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm1
+; NOFMA-NEXT: vaddsubpd %xmm4, %xmm0, %xmm0
+; NOFMA-NEXT: vaddsubpd %xmm0, %xmm1, %xmm0
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain_pd128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm4
+; FMA3-NEXT: vfmaddsub231pd {{.*#+}} xmm0 = (xmm3 * xmm2) +/- xmm0
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_pd128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} xmm0 = (xmm0 * xmm1) +/- xmm4
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} xmm0 = (xmm2 * xmm3) +/- xmm0
+; FMA4-NEXT: retq
+ %AB = fmul contract <2 x double> %A, %B
+ %Sub0 = fsub contract <2 x double> %AB, %E
+ %Add0 = fadd contract <2 x double> %AB, %E
+ %Inner = shufflevector <2 x double> %Sub0, <2 x double> %Add0, <2 x i32> <i32 0, i32 3>
+ %CD = fmul contract <2 x double> %C, %D
+ %Sub1 = fsub contract <2 x double> %CD, %Inner
+ %Add1 = fadd contract <2 x double> %CD, %Inner
+ %Outer = shufflevector <2 x double> %Sub1, <2 x double> %Add1, <2 x i32> <i32 0, i32 3>
+ ret <2 x double> %Outer
+}
+
+; Same with commuted FADD operands and the FADD as the first shuffle operand.
+define <8 x float> @mul_addsub_chain_ps256(<8 x float> %A, <8 x float> %B, <8 x float> %C, <8 x float> %D, <8 x float> %E) {
+; NOFMA-LABEL: mul_addsub_chain_ps256:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulps %ymm1, %ymm0, %ymm0
+; NOFMA-NEXT: vmulps %ymm3, %ymm2, %ymm1
+; NOFMA-NEXT: vaddsubps %ymm4, %ymm0, %ymm0
+; NOFMA-NEXT: vaddsubps %ymm0, %ymm1, %ymm0
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain_ps256:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm4
+; FMA3-NEXT: vfmaddsub231ps {{.*#+}} ymm0 = (ymm3 * ymm2) +/- ymm0
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_ps256:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vfmaddsubps {{.*#+}} ymm0 = (ymm0 * ymm1) +/- ymm4
+; FMA4-NEXT: vfmaddsubps {{.*#+}} ymm0 = (ymm2 * ymm3) +/- ymm0
+; FMA4-NEXT: retq
+ %AB = fmul contract <8 x float> %A, %B
+ %Sub0 = fsub contract <8 x float> %AB, %E
+ %Add0 = fadd contract <8 x float> %E, %AB
+ %Inner = shufflevector <8 x float> %Add0, <8 x float> %Sub0, <8 x i32> <i32 8, i32 1, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
+ %CD = fmul contract <8 x float> %C, %D
+ %Sub1 = fsub contract <8 x float> %CD, %Inner
+ %Add1 = fadd contract <8 x float> %Inner, %CD
+ %Outer = shufflevector <8 x float> %Add1, <8 x float> %Sub1, <8 x i32> <i32 8, i32 1, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
+ ret <8 x float> %Outer
+}
+
+define <8 x double> @mul_addsub_chain_pd512(<8 x double> %A, <8 x double> %B, <8 x double> %C, <8 x double> %D, <8 x double> %E) {
+; NOFMA-LABEL: mul_addsub_chain_pd512:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: pushq %rbp
+; NOFMA-NEXT: .cfi_def_cfa_offset 16
+; NOFMA-NEXT: .cfi_offset %rbp, -16
+; NOFMA-NEXT: movq %rsp, %rbp
+; NOFMA-NEXT: .cfi_def_cfa_register %rbp
+; NOFMA-NEXT: andq $-32, %rsp
+; NOFMA-NEXT: subq $32, %rsp
+; NOFMA-NEXT: vmulpd %ymm3, %ymm1, %ymm1
+; NOFMA-NEXT: vmulpd %ymm2, %ymm0, %ymm0
+; NOFMA-NEXT: vmulpd %ymm7, %ymm5, %ymm2
+; NOFMA-NEXT: vmulpd %ymm6, %ymm4, %ymm3
+; NOFMA-NEXT: vaddsubpd 16(%rbp), %ymm0, %ymm0
+; NOFMA-NEXT: vaddsubpd %ymm0, %ymm3, %ymm0
+; NOFMA-NEXT: vaddsubpd 48(%rbp), %ymm1, %ymm1
+; NOFMA-NEXT: vaddsubpd %ymm1, %ymm2, %ymm1
+; NOFMA-NEXT: movq %rbp, %rsp
+; NOFMA-NEXT: popq %rbp
+; NOFMA-NEXT: .cfi_def_cfa %rsp, 8
+; NOFMA-NEXT: retq
+;
+; FMA3_256-LABEL: mul_addsub_chain_pd512:
+; FMA3_256: # %bb.0:
+; FMA3_256-NEXT: pushq %rbp
+; FMA3_256-NEXT: .cfi_def_cfa_offset 16
+; FMA3_256-NEXT: .cfi_offset %rbp, -16
+; FMA3_256-NEXT: movq %rsp, %rbp
+; FMA3_256-NEXT: .cfi_def_cfa_register %rbp
+; FMA3_256-NEXT: andq $-32, %rsp
+; FMA3_256-NEXT: subq $32, %rsp
+; FMA3_256-NEXT: vfmaddsub213pd {{.*#+}} ymm0 = (ymm2 * ymm0) +/- mem
+; FMA3_256-NEXT: vfmaddsub231pd {{.*#+}} ymm0 = (ymm6 * ymm4) +/- ymm0
+; FMA3_256-NEXT: vfmaddsub213pd {{.*#+}} ymm1 = (ymm3 * ymm1) +/- mem
+; FMA3_256-NEXT: vfmaddsub231pd {{.*#+}} ymm1 = (ymm7 * ymm5) +/- ymm1
+; FMA3_256-NEXT: movq %rbp, %rsp
+; FMA3_256-NEXT: popq %rbp
+; FMA3_256-NEXT: .cfi_def_cfa %rsp, 8
+; FMA3_256-NEXT: retq
+;
+; FMA3_512-LABEL: mul_addsub_chain_pd512:
+; FMA3_512: # %bb.0:
+; FMA3_512-NEXT: vfmaddsub213pd {{.*#+}} zmm0 = (zmm1 * zmm0) +/- zmm4
+; FMA3_512-NEXT: vfmaddsub231pd {{.*#+}} zmm0 = (zmm3 * zmm2) +/- zmm0
+; FMA3_512-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_pd512:
+; FMA4: # %bb.0:
+; FMA4-NEXT: pushq %rbp
+; FMA4-NEXT: .cfi_def_cfa_offset 16
+; FMA4-NEXT: .cfi_offset %rbp, -16
+; FMA4-NEXT: movq %rsp, %rbp
+; FMA4-NEXT: .cfi_def_cfa_register %rbp
+; FMA4-NEXT: andq $-32, %rsp
+; FMA4-NEXT: subq $32, %rsp
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} ymm0 = (ymm0 * ymm2) +/- mem
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} ymm0 = (ymm4 * ymm6) +/- ymm0
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} ymm1 = (ymm1 * ymm3) +/- mem
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} ymm1 = (ymm5 * ymm7) +/- ymm1
+; FMA4-NEXT: movq %rbp, %rsp
+; FMA4-NEXT: popq %rbp
+; FMA4-NEXT: .cfi_def_cfa %rsp, 8
+; FMA4-NEXT: retq
+ %AB = fmul contract <8 x double> %A, %B
+ %Sub0 = fsub contract <8 x double> %AB, %E
+ %Add0 = fadd contract <8 x double> %AB, %E
+ %Inner = shufflevector <8 x double> %Sub0, <8 x double> %Add0, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ %CD = fmul contract <8 x double> %C, %D
+ %Sub1 = fsub contract <8 x double> %CD, %Inner
+ %Add1 = fadd contract <8 x double> %CD, %Inner
+ %Outer = shufflevector <8 x double> %Sub1, <8 x double> %Add1, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ ret <8 x double> %Outer
+}
+
+; Without contract flags the chain can only become two ADDSUBs.
+define <2 x double> @mul_addsub_chain_nocontract_pd128(<2 x double> %A, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E) {
+; NOFMA-LABEL: mul_addsub_chain_nocontract_pd128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm1
+; NOFMA-NEXT: vaddsubpd %xmm4, %xmm0, %xmm0
+; NOFMA-NEXT: vaddsubpd %xmm0, %xmm1, %xmm0
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain_nocontract_pd128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA3-NEXT: vmulpd %xmm3, %xmm2, %xmm1
+; FMA3-NEXT: vaddsubpd %xmm4, %xmm0, %xmm0
+; FMA3-NEXT: vaddsubpd %xmm0, %xmm1, %xmm0
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_nocontract_pd128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; FMA4-NEXT: vmulpd %xmm3, %xmm2, %xmm1
+; FMA4-NEXT: vaddsubpd %xmm4, %xmm0, %xmm0
+; FMA4-NEXT: vaddsubpd %xmm0, %xmm1, %xmm0
+; FMA4-NEXT: retq
+ %AB = fmul <2 x double> %A, %B
+ %Sub0 = fsub <2 x double> %AB, %E
+ %Add0 = fadd <2 x double> %AB, %E
+ %Inner = shufflevector <2 x double> %Sub0, <2 x double> %Add0, <2 x i32> <i32 0, i32 3>
+ %CD = fmul <2 x double> %C, %D
+ %Sub1 = fsub <2 x double> %CD, %Inner
+ %Add1 = fadd <2 x double> %CD, %Inner
+ %Outer = shufflevector <2 x double> %Sub1, <2 x double> %Add1, <2 x i32> <i32 0, i32 3>
+ ret <2 x double> %Outer
+}
+
+; A chain of three multiply-add/subs.
+define <2 x double> @mul_addsub_chain3_pd128(<2 x double> %A, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E, <2 x double> %F, <2 x double> %G) {
+; NOFMA-LABEL: mul_addsub_chain3_pd128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulpd %xmm1, %xmm0, %xmm0
+; NOFMA-NEXT: vmulpd %xmm3, %xmm2, %xmm1
+; NOFMA-NEXT: vmulpd %xmm5, %xmm4, %xmm2
+; NOFMA-NEXT: vaddsubpd %xmm6, %xmm0, %xmm0
+; NOFMA-NEXT: vaddsubpd %xmm0, %xmm1, %xmm0
+; NOFMA-NEXT: vaddsubpd %xmm0, %xmm2, %xmm0
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain3_pd128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm6
+; FMA3-NEXT: vfmaddsub231pd {{.*#+}} xmm0 = (xmm3 * xmm2) +/- xmm0
+; FMA3-NEXT: vfmaddsub231pd {{.*#+}} xmm0 = (xmm5 * xmm4) +/- xmm0
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain3_pd128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} xmm0 = (xmm0 * xmm1) +/- xmm6
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} xmm0 = (xmm2 * xmm3) +/- xmm0
+; FMA4-NEXT: vfmaddsubpd {{.*#+}} xmm0 = (xmm4 * xmm5) +/- xmm0
+; FMA4-NEXT: retq
+ %AB = fmul contract <2 x double> %A, %B
+ %Sub0 = fsub contract <2 x double> %AB, %G
+ %Add0 = fadd contract <2 x double> %AB, %G
+ %Inner0 = shufflevector <2 x double> %Sub0, <2 x double> %Add0, <2 x i32> <i32 0, i32 3>
+ %CD = fmul contract <2 x double> %C, %D
+ %Sub1 = fsub contract <2 x double> %CD, %Inner0
+ %Add1 = fadd contract <2 x double> %CD, %Inner0
+ %Inner1 = shufflevector <2 x double> %Sub1, <2 x double> %Add1, <2 x i32> <i32 0, i32 3>
+ %EF = fmul contract <2 x double> %E, %F
+ %Sub2 = fsub contract <2 x double> %EF, %Inner1
+ %Add2 = fadd contract <2 x double> %EF, %Inner1
+ %Outer = shufflevector <2 x double> %Sub2, <2 x double> %Add2, <2 x i32> <i32 0, i32 3>
+ ret <2 x double> %Outer
+}
+
+; Inner SUBADD feeding an outer ADDSUB.
+define <4 x float> @mul_addsub_chain_subadd_ps128(<4 x float> %A, <4 x float> %B, <4 x float> %C, <4 x float> %D, <4 x float> %E) {
+; NOFMA-LABEL: mul_addsub_chain_subadd_ps128:
+; NOFMA: # %bb.0:
+; NOFMA-NEXT: vmulps %xmm1, %xmm0, %xmm0
+; NOFMA-NEXT: vsubps %xmm4, %xmm0, %xmm1
+; NOFMA-NEXT: vaddps %xmm4, %xmm0, %xmm0
+; NOFMA-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; NOFMA-NEXT: vmulps %xmm3, %xmm2, %xmm1
+; NOFMA-NEXT: vaddsubps %xmm0, %xmm1, %xmm0
+; NOFMA-NEXT: retq
+;
+; FMA3-LABEL: mul_addsub_chain_subadd_ps128:
+; FMA3: # %bb.0:
+; FMA3-NEXT: vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm4
+; FMA3-NEXT: vfmaddsub231ps {{.*#+}} xmm0 = (xmm3 * xmm2) +/- xmm0
+; FMA3-NEXT: retq
+;
+; FMA4-LABEL: mul_addsub_chain_subadd_ps128:
+; FMA4: # %bb.0:
+; FMA4-NEXT: vfmsubaddps {{.*#+}} xmm0 = (xmm0 * xmm1) -/+ xmm4
+; FMA4-NEXT: vfmaddsubps {{.*#+}} xmm0 = (xmm2 * xmm3) +/- xmm0
+; FMA4-NEXT: retq
+ %AB = fmul contract <4 x float> %A, %B
+ %Sub0 = fsub contract <4 x float> %AB, %E
+ %Add0 = fadd contract <4 x float> %AB, %E
+ %Inner = shufflevector <4 x float> %Add0, <4 x float> %Sub0, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+ %CD = fmul contract <4 x float> %C, %D
+ %Sub1 = fsub contract <4 x float> %CD, %Inner
+ %Add1 = fadd contract <4 x float> %CD, %Inner
+ %Outer = shufflevector <4 x float> %Sub1, <4 x float> %Add1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+ ret <4 x float> %Outer
+}
+
+; Onl...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/226429
More information about the llvm-commits
mailing list