[llvm] 3d91765 - [X86] Fold FMSUBADD + FADD into VFCMADDC for fp16 (#222896)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 14 08:47:56 PDT 2026
Author: Rohan Shenoy
Date: 2026-09-14T15:47:49Z
New Revision: 3d91765f54e9c461c3f979a5ea7c302f59303489
URL: https://github.com/llvm/llvm-project/commit/3d91765f54e9c461c3f979a5ea7c302f59303489
DIFF: https://github.com/llvm/llvm-project/commit/3d91765f54e9c461c3f979a5ea7c302f59303489.diff
LOG: [X86] Fold FMSUBADD + FADD into VFCMADDC for fp16 (#222896)
Complex multiply by conjugate and accumulate fp16 (c += a * ~b) is
lowered into vfmsubadd + vaddph, failing to emit native vfcmaddcph in
avx512fp16. We can reuse the pattern for FSUBADD and extend
combineFaddCFmul to perform this fold.
Fixes #216290
Added:
llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b0ac6e611c097..775fd7642f040 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56209,6 +56209,50 @@ static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG,
return Res;
}
+// We try to match the following pattern from FMSUBADD(X, A, M) to lower it
+// into complex conjugate fmadd for fp16 (#216290).
+// for vector of the complex form v <v0r, v0i, v1r, v1i, ...>
+// and 2 complex vectors a, b,
+// X = duplicate real (b) : <b0r, b0r, b1r, b1r, ...>
+// A = a
+// M = FMUL (P, Q)
+// P = adjacent pair swapped (a) : <a0i, a0r, a1i, a1r, ...>
+// Q = duplicate imaginary (b) : <b0i, b0i, b1i, b1i, ...>
+static bool isCFMulFromFMSUBADD(SDValue N, SelectionDAG &DAG, SDValue &A,
+ SDValue &B) {
+ SDValue Op0 = N.getOperand(0);
+ SDValue Op1 = N.getOperand(1);
+ SDValue Op2 = N.getOperand(2);
+
+ auto matchShufflePattern = [&DAG](SDValue V, ArrayRef<int> Pat) {
+ SmallVector<SDValue, 2> Inputs;
+ SmallVector<int, 32> Mask;
+ SmallVector<int, 8> RepeatedMask;
+ MVT VT = V.getSimpleValueType();
+ if (getTargetShuffleInputs(V, Inputs, Mask, DAG) &&
+ is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask) &&
+ isShuffleEquivalent(RepeatedMask, Pat, Inputs[0]))
+ return Inputs[0];
+ return SDValue();
+ };
+ auto matchFMSUBADDPattern = [&](SDValue X, SDValue OpA) {
+ B = matchShufflePattern(X, {0, 0, 2, 2, 4, 4, 6, 6});
+ if (!B)
+ return false;
+ A = OpA;
+ SDValue P = Op2.getOperand(0);
+ SDValue Q = Op2.getOperand(1);
+ auto matchFMulPattern = [&](SDValue P, SDValue Q) {
+ return matchShufflePattern(P, {1, 0, 3, 2, 5, 4, 7, 6}) == A &&
+ matchShufflePattern(Q, {1, 1, 3, 3, 5, 5, 7, 7}) == B;
+ };
+ return matchFMulPattern(P, Q) || matchFMulPattern(Q, P);
+ };
+ // First 2 operands of FMSUBADD are commutable.
+ return Op2.getOpcode() == ISD::FMUL &&
+ (matchFMSUBADDPattern(Op0, Op1) || matchFMSUBADDPattern(Op1, Op0));
+}
+
// Try to combine the following nodes:
// FADD(A, FMA(B, C, 0)) and FADD(A, FMUL(B, C)) to FMA(B, C, A)
static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
@@ -56232,8 +56276,18 @@ static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
SDValue RHS = N->getOperand(1);
bool IsConj;
SDValue FAddOp1, MulOp0, MulOp1;
- auto GetCFmulFrom = [&MulOp0, &MulOp1, &IsConj,
- &IsVectorAllNegativeZero](SDValue N) -> bool {
+ MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2);
+ auto GetCFmulFrom = [&MulOp0, &MulOp1, &IsConj, &DAG,
+ &IsVectorAllNegativeZero, &CVT](SDValue N) -> bool {
+ if (N.getOpcode() == X86ISD::FMSUBADD && N.hasOneUse()) {
+ SDValue A, B;
+ if (!isCFMulFromFMSUBADD(N, DAG, A, B))
+ return false;
+ IsConj = true;
+ MulOp0 = DAG.getBitcast(CVT, A);
+ MulOp1 = DAG.getBitcast(CVT, B);
+ return true;
+ }
if (!N.hasOneUse() || N.getOpcode() != ISD::BITCAST)
return false;
SDValue Op0 = N.getOperand(0);
@@ -56265,7 +56319,6 @@ static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
else
return SDValue();
- MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2);
FAddOp1 = DAG.getBitcast(CVT, FAddOp1);
unsigned NewOp = IsConj ? X86ISD::VFCMADDC : X86ISD::VFMADDC;
// FIXME: How do we handle when fast math flags of FADD are
diff erent from
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
new file mode 100644
index 0000000000000..4d1e7479d0039
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16,avx512vl | FileCheck %s
+
+define <8 x half> @test_128(<8 x half> %a, <8 x half> %b, <8 x half> %acc) {
+; CHECK-LABEL: test_128:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vfcmaddcph %xmm1, %xmm0, %xmm2
+; CHECK-NEXT: vmovaps %xmm2, %xmm0
+; CHECK-NEXT: retq
+ %dup_even_b = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+ %mul_re = fmul contract <8 x half> %a, %dup_even_b
+ %swap_a = shufflevector <8 x half> %a, <8 x half> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+ %dup_odd_b = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+ %mul_im = fmul contract <8 x half> %swap_a, %dup_odd_b
+ %add = fadd contract <8 x half> %mul_re, %mul_im
+ %sub = fsub contract <8 x half> %mul_re, %mul_im
+ %cmul = shufflevector <8 x half> %add, <8 x half> %sub, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ %res = fadd contract <8 x half> %cmul, %acc
+ ret <8 x half> %res
+}
+
+define <16 x half> @test_256(<16 x half> %a, <16 x half> %b, <16 x half> %acc) {
+; CHECK-LABEL: test_256:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vfcmaddcph %ymm1, %ymm0, %ymm2
+; CHECK-NEXT: vmovaps %ymm2, %ymm0
+; CHECK-NEXT: retq
+ %dup_even_b = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+ %mul_re = fmul contract <16 x half> %dup_even_b, %a
+ %swap_a = shufflevector <16 x half> %a, <16 x half> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+ %dup_odd_b = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+ %mul_im = fmul contract <16 x half> %dup_odd_b, %swap_a
+ %add = fadd contract <16 x half> %mul_re, %mul_im
+ %sub = fsub contract <16 x half> %mul_re, %mul_im
+ %cmul = shufflevector <16 x half> %add, <16 x half> %sub, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
+ %res = fadd contract <16 x half> %cmul, %acc
+ ret <16 x half> %res
+}
+
+define <32 x half> @test_512(<32 x half> %a, <32 x half> %b, <32 x half> %acc) {
+; CHECK-LABEL: test_512:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vfcmaddcph %zmm1, %zmm0, %zmm2
+; CHECK-NEXT: vmovaps %zmm2, %zmm0
+; CHECK-NEXT: retq
+ %dup_even_b = shufflevector <32 x half> %b, <32 x half> poison, <32 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14, i32 16, i32 16, i32 18, i32 18, i32 20, i32 20, i32 22, i32 22, i32 24, i32 24, i32 26, i32 26, i32 28, i32 28, i32 30, i32 30>
+ %mul_re = fmul contract <32 x half> %dup_even_b, %a
+ %swap_a = shufflevector <32 x half> %a, <32 x half> poison, <32 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30>
+ %dup_odd_b = shufflevector <32 x half> %b, <32 x half> poison, <32 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15, i32 17, i32 17, i32 19, i32 19, i32 21, i32 21, i32 23, i32 23, i32 25, i32 25, i32 27, i32 27, i32 29, i32 29, i32 31, i32 31>
+ %mul_im = fmul contract <32 x half> %swap_a, %dup_odd_b
+ %add = fadd contract <32 x half> %mul_re, %mul_im
+ %sub = fsub contract <32 x half> %mul_re, %mul_im
+ %cmul = shufflevector <32 x half> %add, <32 x half> %sub, <32 x i32> <i32 0, i32 33, i32 2, i32 35, i32 4, i32 37, i32 6, i32 39, i32 8, i32 41, i32 10, i32 43, i32 12, i32 45, i32 14, i32 47, i32 16, i32 49, i32 18, i32 51, i32 20, i32 53, i32 22, i32 55, i32 24, i32 57, i32 26, i32 59, i32 28, i32 61, i32 30, i32 63>
+ %res = fadd contract <32 x half> %acc, %cmul
+ ret <32 x half> %res
+}
More information about the llvm-commits
mailing list