[llvm] [X86] Fold FMSUBADD + FADD into VFCMADDC for fp16 (PR #222896)
Rohan Shenoy via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 11 02:35:44 PDT 2026
https://github.com/222rohan created https://github.com/llvm/llvm-project/pull/222896
Complex multiply by conjugate and accumulate fp16 (c += a * ~b) is lowered into vfmsubadd + vaddph, failing to emit native vfcmaddcph in avx512fp16. We can reuse the pattern for FSUBADD and extend combineFaddCFmul to perform this fold.
Fixes #216290.
>From bd6b5fba76aff7083d0100962b0ca4f365bba519 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Fri, 11 Sep 2026 11:55:04 +0530
Subject: [PATCH 1/3] before fold
---
.../X86/avx512fp16-combine-fmsubadd-fadd.ll | 77 +++++++++++++++++++
1 file changed, 77 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
new file mode 100644
index 0000000000000..49c88f48bd75b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16,avx512vl | FileCheck %s
+
+define <8 x half> @test_128(<8 x half> %0, <8 x half> %1, <8 x half> %2) {
+; CHECK-LABEL: test_128:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[0,0,2,2,4,5,6,7]
+; CHECK-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,6,6]
+; CHECK-NEXT: vprold $16, %xmm0, %xmm4
+; CHECK-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,1,3,3,4,5,6,7]
+; CHECK-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,7,7]
+; CHECK-NEXT: vmulph %xmm1, %xmm4, %xmm1
+; CHECK-NEXT: vfmsubadd231ph %xmm3, %xmm0, %xmm1
+; CHECK-NEXT: vaddph %xmm2, %xmm1, %xmm0
+; CHECK-NEXT: retq
+entry:
+ %3 = shufflevector <8 x half> %1, <8 x half> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+ %4 = fmul fast <8 x half> %0, %3
+ %5 = shufflevector <8 x half> %0, <8 x half> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+ %6 = shufflevector <8 x half> %1, <8 x half> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+ %7 = fmul fast <8 x half> %5, %6
+ %8 = fadd fast <8 x half> %4, %7
+ %9 = fsub fast <8 x half> %4, %7
+ %10 = shufflevector <8 x half> %8, <8 x half> %9, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ %11 = fadd fast <8 x half> %10, %2
+ ret <8 x half> %11
+}
+
+define <16 x half> @test_256(<16 x half> %0, <16 x half> %1, <16 x half> %2) {
+; CHECK-LABEL: test_256:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpshuflw {{.*#+}} ymm3 = ymm1[0,0,2,2,4,5,6,7,8,8,10,10,12,13,14,15]
+; CHECK-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,4,6,6,8,9,10,11,12,12,14,14]
+; CHECK-NEXT: vprold $16, %ymm0, %ymm4
+; CHECK-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15]
+; CHECK-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,5,5,7,7,8,9,10,11,13,13,15,15]
+; CHECK-NEXT: vmulph %ymm4, %ymm1, %ymm1
+; CHECK-NEXT: vfmsubadd231ph %ymm3, %ymm0, %ymm1
+; CHECK-NEXT: vaddph %ymm2, %ymm1, %ymm0
+; CHECK-NEXT: retq
+entry:
+ %3 = shufflevector <16 x half> %1, <16 x half> poison, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+ %4 = fmul fast <16 x half> %3, %0
+ %5 = shufflevector <16 x half> %0, <16 x half> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+ %6 = shufflevector <16 x half> %1, <16 x half> poison, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+ %7 = fmul fast <16 x half> %6, %5
+ %8 = fadd fast <16 x half> %4, %7
+ %9 = fsub fast <16 x half> %4, %7
+ %10 = shufflevector <16 x half> %8, <16 x half> %9, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
+ %11 = fadd fast <16 x half> %10, %2
+ ret <16 x half> %11
+}
+
+define <32 x half> @test_512(<32 x half> %0, <32 x half> %1, <32 x half> %2) {
+; CHECK-LABEL: test_512:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpshuflw {{.*#+}} zmm3 = zmm1[0,0,2,2,4,5,6,7,8,8,10,10,12,13,14,15,16,16,18,18,20,21,22,23,24,24,26,26,28,29,30,31]
+; CHECK-NEXT: vpshufhw {{.*#+}} zmm3 = zmm3[0,1,2,3,4,4,6,6,8,9,10,11,12,12,14,14,16,17,18,19,20,20,22,22,24,25,26,27,28,28,30,30]
+; CHECK-NEXT: vprold $16, %zmm0, %zmm4
+; CHECK-NEXT: vpshuflw {{.*#+}} zmm1 = zmm1[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15,17,17,19,19,20,21,22,23,25,25,27,27,28,29,30,31]
+; CHECK-NEXT: vpshufhw {{.*#+}} zmm1 = zmm1[0,1,2,3,5,5,7,7,8,9,10,11,13,13,15,15,16,17,18,19,21,21,23,23,24,25,26,27,29,29,31,31]
+; CHECK-NEXT: vmulph %zmm1, %zmm4, %zmm1
+; CHECK-NEXT: vfmsubadd231ph %zmm3, %zmm0, %zmm1
+; CHECK-NEXT: vaddph %zmm1, %zmm2, %zmm0
+; CHECK-NEXT: retq
+entry:
+ %3 = shufflevector <32 x half> %1, <32 x half> poison, <32 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14, i32 16, i32 16, i32 18, i32 18, i32 20, i32 20, i32 22, i32 22, i32 24, i32 24, i32 26, i32 26, i32 28, i32 28, i32 30, i32 30>
+ %4 = fmul fast <32 x half> %3, %0
+ %5 = shufflevector <32 x half> %0, <32 x half> poison, <32 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30>
+ %6 = shufflevector <32 x half> %1, <32 x half> poison, <32 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15, i32 17, i32 17, i32 19, i32 19, i32 21, i32 21, i32 23, i32 23, i32 25, i32 25, i32 27, i32 27, i32 29, i32 29, i32 31, i32 31>
+ %7 = fmul fast <32 x half> %5, %6
+ %8 = fadd fast <32 x half> %4, %7
+ %9 = fsub fast <32 x half> %4, %7
+ %10 = shufflevector <32 x half> %8, <32 x half> %9, <32 x i32> <i32 0, i32 33, i32 2, i32 35, i32 4, i32 37, i32 6, i32 39, i32 8, i32 41, i32 10, i32 43, i32 12, i32 45, i32 14, i32 47, i32 16, i32 49, i32 18, i32 51, i32 20, i32 53, i32 22, i32 55, i32 24, i32 57, i32 26, i32 59, i32 28, i32 61, i32 30, i32 63>
+ %11 = fadd fast <32 x half> %2, %10
+ ret <32 x half> %11
+}
>From fa483ab4f3cab89104656e5500db20eacff326b2 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Fri, 11 Sep 2026 11:56:51 +0530
Subject: [PATCH 2/3] add fold to vfcmaddcph
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 85 ++++++++++++++++++++++++-
1 file changed, 82 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index a41cb2914660c..2520fa1c4a0b5 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56066,6 +56066,76 @@ static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG,
return Res;
}
+// We try to match the following pattern from FMSUBADD(X, A, M) to lower it
+// into complex conjugate fmadd for fp16 (#216290).
+// for vector of the complex form v <v0r, v0i, v1r, v1i, ...>
+// and 2 complex vectors a, b,
+// X = duplicate real (b) : <b0r, b0r, b1r, b1r, ...>
+// A = a
+// M = FMUL (P, Q)
+// P = adjacent pair swapped (a) : <a0i, a0r, a1i, a1r, ...>
+// Q = duplicate imaginary (b) : <b0i, b0i, b1i, b1i, ...>
+static bool isCFMulFromFMSUBADD(SDValue N, SelectionDAG &DAG, SDValue &A,
+ SDValue &B) {
+ // isEven: Shuffle Mask <0, 0, 2, 2, ...> V
+ // else, Shuffle Mask <1, 1, 3, 3, ...> V
+ auto isSplat2EvenOrOddMask = [](ArrayRef<int> Mask, bool isEven) {
+ unsigned Size = Mask.size();
+ if (Size % 2 != 0)
+ return false;
+ unsigned Start = isEven ? 0 : 1;
+ for (unsigned i = 0; i < Size; i += 2) {
+ if (Mask[i] != (int)(Start + i) || Mask[i + 1] != (int)(Start + i))
+ return false;
+ }
+ return true;
+ };
+
+ // Shuffle Mask <1, 0, 3, 2, ...> V
+ auto isSwapAdjPairMask = [](ArrayRef<int> Mask) {
+ unsigned Size = Mask.size();
+ if (Size % 2 != 0)
+ return false;
+ for (unsigned i = 0; i < Size; i += 2) {
+ if (Mask[i] != (int)(i + 1) || Mask[i + 1] != (int)i)
+ return false;
+ }
+ return true;
+ };
+
+ SDValue Op0 = N.getOperand(0);
+ SDValue Op1 = N.getOperand(1);
+ SDValue Op2 = N.getOperand(2);
+
+ SmallVector<SDValue, 2> Inputs;
+ SmallVector<int, 32> Mask;
+ auto matchFMSUBADDPattern = [&](SDValue X, SDValue OpA) -> bool {
+ Inputs.clear();
+ Mask.clear();
+ A = OpA;
+
+ if (!getTargetShuffleInputs(X, Inputs, Mask, DAG) ||
+ !isSplat2EvenOrOddMask(Mask, true))
+ return false;
+ B = Inputs[0];
+
+ if (Op2.getOpcode() != ISD::FMUL)
+ return false;
+ SDValue P = Op2.getOperand(0);
+ SDValue Q = Op2.getOperand(1);
+ auto matchFMulPattern = [&](SDValue P, SDValue Q) {
+ return getTargetShuffleInputs(P, Inputs, Mask, DAG) &&
+ isSwapAdjPairMask(Mask) && Inputs[0] == A &&
+ getTargetShuffleInputs(Q, Inputs, Mask, DAG) &&
+ isSplat2EvenOrOddMask(Mask, false) && Inputs[0] == B;
+ };
+ return matchFMulPattern(P, Q) || matchFMulPattern(Q, P);
+ };
+
+ // First 2 operands of FMSUBADD are commutable.
+ return matchFMSUBADDPattern(Op0, Op1) || matchFMSUBADDPattern(Op1, Op0);
+}
+
// Try to combine the following nodes:
// FADD(A, FMA(B, C, 0)) and FADD(A, FMUL(B, C)) to FMA(B, C, A)
static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
@@ -56089,8 +56159,18 @@ static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
SDValue RHS = N->getOperand(1);
bool IsConj;
SDValue FAddOp1, MulOp0, MulOp1;
- auto GetCFmulFrom = [&MulOp0, &MulOp1, &IsConj,
- &IsVectorAllNegativeZero](SDValue N) -> bool {
+ MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2);
+ auto GetCFmulFrom = [&MulOp0, &MulOp1, &IsConj, &DAG,
+ &IsVectorAllNegativeZero, &CVT](SDValue N) -> bool {
+ if (N.getOpcode() == X86ISD::FMSUBADD && N.hasOneUse()) {
+ SDValue A, B;
+ if (!isCFMulFromFMSUBADD(N, DAG, A, B))
+ return false;
+ IsConj = true;
+ MulOp0 = DAG.getBitcast(CVT, A);
+ MulOp1 = DAG.getBitcast(CVT, B);
+ return true;
+ }
if (!N.hasOneUse() || N.getOpcode() != ISD::BITCAST)
return false;
SDValue Op0 = N.getOperand(0);
@@ -56122,7 +56202,6 @@ static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
else
return SDValue();
- MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2);
FAddOp1 = DAG.getBitcast(CVT, FAddOp1);
unsigned NewOp = IsConj ? X86ISD::VFCMADDC : X86ISD::VFMADDC;
// FIXME: How do we handle when fast math flags of FADD are different from
>From 606197a3e3b16237e9f93dd1908092c345b81393 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Fri, 11 Sep 2026 14:52:17 +0530
Subject: [PATCH 3/3] post fold
---
.../X86/avx512fp16-combine-fmsubadd-fadd.ll | 30 ++++---------------
1 file changed, 6 insertions(+), 24 deletions(-)
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
index 49c88f48bd75b..abcd6f4e4eea3 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
@@ -4,14 +4,8 @@
define <8 x half> @test_128(<8 x half> %0, <8 x half> %1, <8 x half> %2) {
; CHECK-LABEL: test_128:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[0,0,2,2,4,5,6,7]
-; CHECK-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,6,6]
-; CHECK-NEXT: vprold $16, %xmm0, %xmm4
-; CHECK-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,1,3,3,4,5,6,7]
-; CHECK-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,7,7]
-; CHECK-NEXT: vmulph %xmm1, %xmm4, %xmm1
-; CHECK-NEXT: vfmsubadd231ph %xmm3, %xmm0, %xmm1
-; CHECK-NEXT: vaddph %xmm2, %xmm1, %xmm0
+; CHECK-NEXT: vfcmaddcph %xmm1, %xmm0, %xmm2
+; CHECK-NEXT: vmovaps %xmm2, %xmm0
; CHECK-NEXT: retq
entry:
%3 = shufflevector <8 x half> %1, <8 x half> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
@@ -29,14 +23,8 @@ entry:
define <16 x half> @test_256(<16 x half> %0, <16 x half> %1, <16 x half> %2) {
; CHECK-LABEL: test_256:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vpshuflw {{.*#+}} ymm3 = ymm1[0,0,2,2,4,5,6,7,8,8,10,10,12,13,14,15]
-; CHECK-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,4,6,6,8,9,10,11,12,12,14,14]
-; CHECK-NEXT: vprold $16, %ymm0, %ymm4
-; CHECK-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15]
-; CHECK-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,5,5,7,7,8,9,10,11,13,13,15,15]
-; CHECK-NEXT: vmulph %ymm4, %ymm1, %ymm1
-; CHECK-NEXT: vfmsubadd231ph %ymm3, %ymm0, %ymm1
-; CHECK-NEXT: vaddph %ymm2, %ymm1, %ymm0
+; CHECK-NEXT: vfcmaddcph %ymm1, %ymm0, %ymm2
+; CHECK-NEXT: vmovaps %ymm2, %ymm0
; CHECK-NEXT: retq
entry:
%3 = shufflevector <16 x half> %1, <16 x half> poison, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
@@ -54,14 +42,8 @@ entry:
define <32 x half> @test_512(<32 x half> %0, <32 x half> %1, <32 x half> %2) {
; CHECK-LABEL: test_512:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vpshuflw {{.*#+}} zmm3 = zmm1[0,0,2,2,4,5,6,7,8,8,10,10,12,13,14,15,16,16,18,18,20,21,22,23,24,24,26,26,28,29,30,31]
-; CHECK-NEXT: vpshufhw {{.*#+}} zmm3 = zmm3[0,1,2,3,4,4,6,6,8,9,10,11,12,12,14,14,16,17,18,19,20,20,22,22,24,25,26,27,28,28,30,30]
-; CHECK-NEXT: vprold $16, %zmm0, %zmm4
-; CHECK-NEXT: vpshuflw {{.*#+}} zmm1 = zmm1[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15,17,17,19,19,20,21,22,23,25,25,27,27,28,29,30,31]
-; CHECK-NEXT: vpshufhw {{.*#+}} zmm1 = zmm1[0,1,2,3,5,5,7,7,8,9,10,11,13,13,15,15,16,17,18,19,21,21,23,23,24,25,26,27,29,29,31,31]
-; CHECK-NEXT: vmulph %zmm1, %zmm4, %zmm1
-; CHECK-NEXT: vfmsubadd231ph %zmm3, %zmm0, %zmm1
-; CHECK-NEXT: vaddph %zmm1, %zmm2, %zmm0
+; CHECK-NEXT: vfcmaddcph %zmm1, %zmm0, %zmm2
+; CHECK-NEXT: vmovaps %zmm2, %zmm0
; CHECK-NEXT: retq
entry:
%3 = shufflevector <32 x half> %1, <32 x half> poison, <32 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14, i32 16, i32 16, i32 18, i32 18, i32 20, i32 20, i32 22, i32 22, i32 24, i32 24, i32 26, i32 26, i32 28, i32 28, i32 30, i32 30>
More information about the llvm-commits
mailing list