[llvm] [X86] Fold FMSUBADD + FADD into VFCMADDC for fp16 (PR #222896)
Rohan Shenoy via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 14 04:02:42 PDT 2026
https://github.com/222rohan updated https://github.com/llvm/llvm-project/pull/222896
>From bd6b5fba76aff7083d0100962b0ca4f365bba519 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Fri, 11 Sep 2026 11:55:04 +0530
Subject: [PATCH 1/6] before fold
---
.../X86/avx512fp16-combine-fmsubadd-fadd.ll | 77 +++++++++++++++++++
1 file changed, 77 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
new file mode 100644
index 0000000000000..49c88f48bd75b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16,avx512vl | FileCheck %s
+
+define <8 x half> @test_128(<8 x half> %0, <8 x half> %1, <8 x half> %2) {
+; CHECK-LABEL: test_128:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[0,0,2,2,4,5,6,7]
+; CHECK-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,6,6]
+; CHECK-NEXT: vprold $16, %xmm0, %xmm4
+; CHECK-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,1,3,3,4,5,6,7]
+; CHECK-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,7,7]
+; CHECK-NEXT: vmulph %xmm1, %xmm4, %xmm1
+; CHECK-NEXT: vfmsubadd231ph %xmm3, %xmm0, %xmm1
+; CHECK-NEXT: vaddph %xmm2, %xmm1, %xmm0
+; CHECK-NEXT: retq
+entry:
+ %3 = shufflevector <8 x half> %1, <8 x half> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+ %4 = fmul fast <8 x half> %0, %3
+ %5 = shufflevector <8 x half> %0, <8 x half> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+ %6 = shufflevector <8 x half> %1, <8 x half> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+ %7 = fmul fast <8 x half> %5, %6
+ %8 = fadd fast <8 x half> %4, %7
+ %9 = fsub fast <8 x half> %4, %7
+ %10 = shufflevector <8 x half> %8, <8 x half> %9, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ %11 = fadd fast <8 x half> %10, %2
+ ret <8 x half> %11
+}
+
+define <16 x half> @test_256(<16 x half> %0, <16 x half> %1, <16 x half> %2) {
+; CHECK-LABEL: test_256:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpshuflw {{.*#+}} ymm3 = ymm1[0,0,2,2,4,5,6,7,8,8,10,10,12,13,14,15]
+; CHECK-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,4,6,6,8,9,10,11,12,12,14,14]
+; CHECK-NEXT: vprold $16, %ymm0, %ymm4
+; CHECK-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15]
+; CHECK-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,5,5,7,7,8,9,10,11,13,13,15,15]
+; CHECK-NEXT: vmulph %ymm4, %ymm1, %ymm1
+; CHECK-NEXT: vfmsubadd231ph %ymm3, %ymm0, %ymm1
+; CHECK-NEXT: vaddph %ymm2, %ymm1, %ymm0
+; CHECK-NEXT: retq
+entry:
+ %3 = shufflevector <16 x half> %1, <16 x half> poison, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+ %4 = fmul fast <16 x half> %3, %0
+ %5 = shufflevector <16 x half> %0, <16 x half> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+ %6 = shufflevector <16 x half> %1, <16 x half> poison, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+ %7 = fmul fast <16 x half> %6, %5
+ %8 = fadd fast <16 x half> %4, %7
+ %9 = fsub fast <16 x half> %4, %7
+ %10 = shufflevector <16 x half> %8, <16 x half> %9, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
+ %11 = fadd fast <16 x half> %10, %2
+ ret <16 x half> %11
+}
+
+define <32 x half> @test_512(<32 x half> %0, <32 x half> %1, <32 x half> %2) {
+; CHECK-LABEL: test_512:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpshuflw {{.*#+}} zmm3 = zmm1[0,0,2,2,4,5,6,7,8,8,10,10,12,13,14,15,16,16,18,18,20,21,22,23,24,24,26,26,28,29,30,31]
+; CHECK-NEXT: vpshufhw {{.*#+}} zmm3 = zmm3[0,1,2,3,4,4,6,6,8,9,10,11,12,12,14,14,16,17,18,19,20,20,22,22,24,25,26,27,28,28,30,30]
+; CHECK-NEXT: vprold $16, %zmm0, %zmm4
+; CHECK-NEXT: vpshuflw {{.*#+}} zmm1 = zmm1[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15,17,17,19,19,20,21,22,23,25,25,27,27,28,29,30,31]
+; CHECK-NEXT: vpshufhw {{.*#+}} zmm1 = zmm1[0,1,2,3,5,5,7,7,8,9,10,11,13,13,15,15,16,17,18,19,21,21,23,23,24,25,26,27,29,29,31,31]
+; CHECK-NEXT: vmulph %zmm1, %zmm4, %zmm1
+; CHECK-NEXT: vfmsubadd231ph %zmm3, %zmm0, %zmm1
+; CHECK-NEXT: vaddph %zmm1, %zmm2, %zmm0
+; CHECK-NEXT: retq
+entry:
+ %3 = shufflevector <32 x half> %1, <32 x half> poison, <32 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14, i32 16, i32 16, i32 18, i32 18, i32 20, i32 20, i32 22, i32 22, i32 24, i32 24, i32 26, i32 26, i32 28, i32 28, i32 30, i32 30>
+ %4 = fmul fast <32 x half> %3, %0
+ %5 = shufflevector <32 x half> %0, <32 x half> poison, <32 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30>
+ %6 = shufflevector <32 x half> %1, <32 x half> poison, <32 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15, i32 17, i32 17, i32 19, i32 19, i32 21, i32 21, i32 23, i32 23, i32 25, i32 25, i32 27, i32 27, i32 29, i32 29, i32 31, i32 31>
+ %7 = fmul fast <32 x half> %5, %6
+ %8 = fadd fast <32 x half> %4, %7
+ %9 = fsub fast <32 x half> %4, %7
+ %10 = shufflevector <32 x half> %8, <32 x half> %9, <32 x i32> <i32 0, i32 33, i32 2, i32 35, i32 4, i32 37, i32 6, i32 39, i32 8, i32 41, i32 10, i32 43, i32 12, i32 45, i32 14, i32 47, i32 16, i32 49, i32 18, i32 51, i32 20, i32 53, i32 22, i32 55, i32 24, i32 57, i32 26, i32 59, i32 28, i32 61, i32 30, i32 63>
+ %11 = fadd fast <32 x half> %2, %10
+ ret <32 x half> %11
+}
>From fa483ab4f3cab89104656e5500db20eacff326b2 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Fri, 11 Sep 2026 11:56:51 +0530
Subject: [PATCH 2/6] add fold to vfcmaddcph
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 85 ++++++++++++++++++++++++-
1 file changed, 82 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index a41cb2914660c..2520fa1c4a0b5 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56066,6 +56066,76 @@ static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG,
return Res;
}
+// We try to match the following pattern from FMSUBADD(X, A, M) to lower it
+// into complex conjugate fmadd for fp16 (#216290).
+// for vector of the complex form v <v0r, v0i, v1r, v1i, ...>
+// and 2 complex vectors a, b,
+// X = duplicate real (b) : <b0r, b0r, b1r, b1r, ...>
+// A = a
+// M = FMUL (P, Q)
+// P = adjacent pair swapped (a) : <a0i, a0r, a1i, a1r, ...>
+// Q = duplicate imaginary (b) : <b0i, b0i, b1i, b1i, ...>
+static bool isCFMulFromFMSUBADD(SDValue N, SelectionDAG &DAG, SDValue &A,
+ SDValue &B) {
+ // isEven: Shuffle Mask <0, 0, 2, 2, ...> V
+ // else, Shuffle Mask <1, 1, 3, 3, ...> V
+ auto isSplat2EvenOrOddMask = [](ArrayRef<int> Mask, bool isEven) {
+ unsigned Size = Mask.size();
+ if (Size % 2 != 0)
+ return false;
+ unsigned Start = isEven ? 0 : 1;
+ for (unsigned i = 0; i < Size; i += 2) {
+ if (Mask[i] != (int)(Start + i) || Mask[i + 1] != (int)(Start + i))
+ return false;
+ }
+ return true;
+ };
+
+ // Shuffle Mask <1, 0, 3, 2, ...> V
+ auto isSwapAdjPairMask = [](ArrayRef<int> Mask) {
+ unsigned Size = Mask.size();
+ if (Size % 2 != 0)
+ return false;
+ for (unsigned i = 0; i < Size; i += 2) {
+ if (Mask[i] != (int)(i + 1) || Mask[i + 1] != (int)i)
+ return false;
+ }
+ return true;
+ };
+
+ SDValue Op0 = N.getOperand(0);
+ SDValue Op1 = N.getOperand(1);
+ SDValue Op2 = N.getOperand(2);
+
+ SmallVector<SDValue, 2> Inputs;
+ SmallVector<int, 32> Mask;
+ auto matchFMSUBADDPattern = [&](SDValue X, SDValue OpA) -> bool {
+ Inputs.clear();
+ Mask.clear();
+ A = OpA;
+
+ if (!getTargetShuffleInputs(X, Inputs, Mask, DAG) ||
+ !isSplat2EvenOrOddMask(Mask, true))
+ return false;
+ B = Inputs[0];
+
+ if (Op2.getOpcode() != ISD::FMUL)
+ return false;
+ SDValue P = Op2.getOperand(0);
+ SDValue Q = Op2.getOperand(1);
+ auto matchFMulPattern = [&](SDValue P, SDValue Q) {
+ return getTargetShuffleInputs(P, Inputs, Mask, DAG) &&
+ isSwapAdjPairMask(Mask) && Inputs[0] == A &&
+ getTargetShuffleInputs(Q, Inputs, Mask, DAG) &&
+ isSplat2EvenOrOddMask(Mask, false) && Inputs[0] == B;
+ };
+ return matchFMulPattern(P, Q) || matchFMulPattern(Q, P);
+ };
+
+ // First 2 operands of FMSUBADD are commutable.
+ return matchFMSUBADDPattern(Op0, Op1) || matchFMSUBADDPattern(Op1, Op0);
+}
+
// Try to combine the following nodes:
// FADD(A, FMA(B, C, 0)) and FADD(A, FMUL(B, C)) to FMA(B, C, A)
static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
@@ -56089,8 +56159,18 @@ static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
SDValue RHS = N->getOperand(1);
bool IsConj;
SDValue FAddOp1, MulOp0, MulOp1;
- auto GetCFmulFrom = [&MulOp0, &MulOp1, &IsConj,
- &IsVectorAllNegativeZero](SDValue N) -> bool {
+ MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2);
+ auto GetCFmulFrom = [&MulOp0, &MulOp1, &IsConj, &DAG,
+ &IsVectorAllNegativeZero, &CVT](SDValue N) -> bool {
+ if (N.getOpcode() == X86ISD::FMSUBADD && N.hasOneUse()) {
+ SDValue A, B;
+ if (!isCFMulFromFMSUBADD(N, DAG, A, B))
+ return false;
+ IsConj = true;
+ MulOp0 = DAG.getBitcast(CVT, A);
+ MulOp1 = DAG.getBitcast(CVT, B);
+ return true;
+ }
if (!N.hasOneUse() || N.getOpcode() != ISD::BITCAST)
return false;
SDValue Op0 = N.getOperand(0);
@@ -56122,7 +56202,6 @@ static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
else
return SDValue();
- MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2);
FAddOp1 = DAG.getBitcast(CVT, FAddOp1);
unsigned NewOp = IsConj ? X86ISD::VFCMADDC : X86ISD::VFMADDC;
// FIXME: How do we handle when fast math flags of FADD are different from
>From 606197a3e3b16237e9f93dd1908092c345b81393 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Fri, 11 Sep 2026 14:52:17 +0530
Subject: [PATCH 3/6] post fold
---
.../X86/avx512fp16-combine-fmsubadd-fadd.ll | 30 ++++---------------
1 file changed, 6 insertions(+), 24 deletions(-)
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
index 49c88f48bd75b..abcd6f4e4eea3 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
@@ -4,14 +4,8 @@
define <8 x half> @test_128(<8 x half> %0, <8 x half> %1, <8 x half> %2) {
; CHECK-LABEL: test_128:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[0,0,2,2,4,5,6,7]
-; CHECK-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,6,6]
-; CHECK-NEXT: vprold $16, %xmm0, %xmm4
-; CHECK-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,1,3,3,4,5,6,7]
-; CHECK-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,7,7]
-; CHECK-NEXT: vmulph %xmm1, %xmm4, %xmm1
-; CHECK-NEXT: vfmsubadd231ph %xmm3, %xmm0, %xmm1
-; CHECK-NEXT: vaddph %xmm2, %xmm1, %xmm0
+; CHECK-NEXT: vfcmaddcph %xmm1, %xmm0, %xmm2
+; CHECK-NEXT: vmovaps %xmm2, %xmm0
; CHECK-NEXT: retq
entry:
%3 = shufflevector <8 x half> %1, <8 x half> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
@@ -29,14 +23,8 @@ entry:
define <16 x half> @test_256(<16 x half> %0, <16 x half> %1, <16 x half> %2) {
; CHECK-LABEL: test_256:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vpshuflw {{.*#+}} ymm3 = ymm1[0,0,2,2,4,5,6,7,8,8,10,10,12,13,14,15]
-; CHECK-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,4,6,6,8,9,10,11,12,12,14,14]
-; CHECK-NEXT: vprold $16, %ymm0, %ymm4
-; CHECK-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15]
-; CHECK-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,5,5,7,7,8,9,10,11,13,13,15,15]
-; CHECK-NEXT: vmulph %ymm4, %ymm1, %ymm1
-; CHECK-NEXT: vfmsubadd231ph %ymm3, %ymm0, %ymm1
-; CHECK-NEXT: vaddph %ymm2, %ymm1, %ymm0
+; CHECK-NEXT: vfcmaddcph %ymm1, %ymm0, %ymm2
+; CHECK-NEXT: vmovaps %ymm2, %ymm0
; CHECK-NEXT: retq
entry:
%3 = shufflevector <16 x half> %1, <16 x half> poison, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
@@ -54,14 +42,8 @@ entry:
define <32 x half> @test_512(<32 x half> %0, <32 x half> %1, <32 x half> %2) {
; CHECK-LABEL: test_512:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vpshuflw {{.*#+}} zmm3 = zmm1[0,0,2,2,4,5,6,7,8,8,10,10,12,13,14,15,16,16,18,18,20,21,22,23,24,24,26,26,28,29,30,31]
-; CHECK-NEXT: vpshufhw {{.*#+}} zmm3 = zmm3[0,1,2,3,4,4,6,6,8,9,10,11,12,12,14,14,16,17,18,19,20,20,22,22,24,25,26,27,28,28,30,30]
-; CHECK-NEXT: vprold $16, %zmm0, %zmm4
-; CHECK-NEXT: vpshuflw {{.*#+}} zmm1 = zmm1[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15,17,17,19,19,20,21,22,23,25,25,27,27,28,29,30,31]
-; CHECK-NEXT: vpshufhw {{.*#+}} zmm1 = zmm1[0,1,2,3,5,5,7,7,8,9,10,11,13,13,15,15,16,17,18,19,21,21,23,23,24,25,26,27,29,29,31,31]
-; CHECK-NEXT: vmulph %zmm1, %zmm4, %zmm1
-; CHECK-NEXT: vfmsubadd231ph %zmm3, %zmm0, %zmm1
-; CHECK-NEXT: vaddph %zmm1, %zmm2, %zmm0
+; CHECK-NEXT: vfcmaddcph %zmm1, %zmm0, %zmm2
+; CHECK-NEXT: vmovaps %zmm2, %zmm0
; CHECK-NEXT: retq
entry:
%3 = shufflevector <32 x half> %1, <32 x half> poison, <32 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14, i32 16, i32 16, i32 18, i32 18, i32 20, i32 20, i32 22, i32 22, i32 24, i32 24, i32 26, i32 26, i32 28, i32 28, i32 30, i32 30>
>From eb6cf50535075b6cb9ef7e53a4d3d366023f4a80 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Sat, 12 Sep 2026 20:53:04 +0530
Subject: [PATCH 4/6] comments
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 56 +++++---------
.../X86/avx512fp16-combine-fmsubadd-fadd.ll | 75 +++++++++----------
2 files changed, 53 insertions(+), 78 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 2520fa1c4a0b5..9259f0475d784 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56077,61 +56077,39 @@ static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG,
// Q = duplicate imaginary (b) : <b0i, b0i, b1i, b1i, ...>
static bool isCFMulFromFMSUBADD(SDValue N, SelectionDAG &DAG, SDValue &A,
SDValue &B) {
- // isEven: Shuffle Mask <0, 0, 2, 2, ...> V
- // else, Shuffle Mask <1, 1, 3, 3, ...> V
- auto isSplat2EvenOrOddMask = [](ArrayRef<int> Mask, bool isEven) {
- unsigned Size = Mask.size();
- if (Size % 2 != 0)
- return false;
- unsigned Start = isEven ? 0 : 1;
- for (unsigned i = 0; i < Size; i += 2) {
- if (Mask[i] != (int)(Start + i) || Mask[i + 1] != (int)(Start + i))
- return false;
- }
- return true;
- };
-
- // Shuffle Mask <1, 0, 3, 2, ...> V
- auto isSwapAdjPairMask = [](ArrayRef<int> Mask) {
- unsigned Size = Mask.size();
- if (Size % 2 != 0)
- return false;
- for (unsigned i = 0; i < Size; i += 2) {
- if (Mask[i] != (int)(i + 1) || Mask[i + 1] != (int)i)
- return false;
- }
- return true;
- };
-
SDValue Op0 = N.getOperand(0);
SDValue Op1 = N.getOperand(1);
SDValue Op2 = N.getOperand(2);
-
SmallVector<SDValue, 2> Inputs;
- SmallVector<int, 32> Mask;
- auto matchFMSUBADDPattern = [&](SDValue X, SDValue OpA) -> bool {
+
+ auto matchShufflePattern = [&DAG, &Inputs](SDValue V, ArrayRef<int> Pat) -> bool {
+ SmallVector<int, 32> Mask;
+ SmallVector<int, 8> RepeatedMask;
Inputs.clear();
- Mask.clear();
- A = OpA;
+ if (!getTargetShuffleInputs(V, Inputs, Mask, DAG))
+ return false;
+ MVT VT = V.getSimpleValueType();
+ return is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask)
+ && isShuffleEquivalent(RepeatedMask, Pat, Inputs[0]);
+ };
- if (!getTargetShuffleInputs(X, Inputs, Mask, DAG) ||
- !isSplat2EvenOrOddMask(Mask, true))
+ auto matchFMSUBADDPattern = [&](SDValue X, SDValue OpA) -> bool {
+ if (!matchShufflePattern(X, {0, 0, 2, 2, 4, 4, 6, 6}))
return false;
B = Inputs[0];
-
+ A = OpA;
if (Op2.getOpcode() != ISD::FMUL)
return false;
SDValue P = Op2.getOperand(0);
SDValue Q = Op2.getOperand(1);
auto matchFMulPattern = [&](SDValue P, SDValue Q) {
- return getTargetShuffleInputs(P, Inputs, Mask, DAG) &&
- isSwapAdjPairMask(Mask) && Inputs[0] == A &&
- getTargetShuffleInputs(Q, Inputs, Mask, DAG) &&
- isSplat2EvenOrOddMask(Mask, false) && Inputs[0] == B;
+ return matchShufflePattern(P, {1, 0, 3, 2, 5, 4, 7, 6})
+ && Inputs[0] == A
+ && matchShufflePattern(Q, {1, 1, 3, 3, 5, 5, 7, 7})
+ && Inputs[0] == B;
};
return matchFMulPattern(P, Q) || matchFMulPattern(Q, P);
};
-
// First 2 operands of FMSUBADD are commutable.
return matchFMSUBADDPattern(Op0, Op1) || matchFMSUBADDPattern(Op1, Op0);
}
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
index abcd6f4e4eea3..4d1e7479d0039 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
@@ -1,59 +1,56 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16,avx512vl | FileCheck %s
-define <8 x half> @test_128(<8 x half> %0, <8 x half> %1, <8 x half> %2) {
+define <8 x half> @test_128(<8 x half> %a, <8 x half> %b, <8 x half> %acc) {
; CHECK-LABEL: test_128:
-; CHECK: # %bb.0: # %entry
+; CHECK: # %bb.0:
; CHECK-NEXT: vfcmaddcph %xmm1, %xmm0, %xmm2
; CHECK-NEXT: vmovaps %xmm2, %xmm0
; CHECK-NEXT: retq
-entry:
- %3 = shufflevector <8 x half> %1, <8 x half> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
- %4 = fmul fast <8 x half> %0, %3
- %5 = shufflevector <8 x half> %0, <8 x half> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
- %6 = shufflevector <8 x half> %1, <8 x half> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
- %7 = fmul fast <8 x half> %5, %6
- %8 = fadd fast <8 x half> %4, %7
- %9 = fsub fast <8 x half> %4, %7
- %10 = shufflevector <8 x half> %8, <8 x half> %9, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
- %11 = fadd fast <8 x half> %10, %2
- ret <8 x half> %11
+ %dup_even_b = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+ %mul_re = fmul contract <8 x half> %a, %dup_even_b
+ %swap_a = shufflevector <8 x half> %a, <8 x half> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+ %dup_odd_b = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+ %mul_im = fmul contract <8 x half> %swap_a, %dup_odd_b
+ %add = fadd contract <8 x half> %mul_re, %mul_im
+ %sub = fsub contract <8 x half> %mul_re, %mul_im
+ %cmul = shufflevector <8 x half> %add, <8 x half> %sub, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ %res = fadd contract <8 x half> %cmul, %acc
+ ret <8 x half> %res
}
-define <16 x half> @test_256(<16 x half> %0, <16 x half> %1, <16 x half> %2) {
+define <16 x half> @test_256(<16 x half> %a, <16 x half> %b, <16 x half> %acc) {
; CHECK-LABEL: test_256:
-; CHECK: # %bb.0: # %entry
+; CHECK: # %bb.0:
; CHECK-NEXT: vfcmaddcph %ymm1, %ymm0, %ymm2
; CHECK-NEXT: vmovaps %ymm2, %ymm0
; CHECK-NEXT: retq
-entry:
- %3 = shufflevector <16 x half> %1, <16 x half> poison, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
- %4 = fmul fast <16 x half> %3, %0
- %5 = shufflevector <16 x half> %0, <16 x half> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
- %6 = shufflevector <16 x half> %1, <16 x half> poison, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
- %7 = fmul fast <16 x half> %6, %5
- %8 = fadd fast <16 x half> %4, %7
- %9 = fsub fast <16 x half> %4, %7
- %10 = shufflevector <16 x half> %8, <16 x half> %9, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
- %11 = fadd fast <16 x half> %10, %2
- ret <16 x half> %11
+ %dup_even_b = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+ %mul_re = fmul contract <16 x half> %dup_even_b, %a
+ %swap_a = shufflevector <16 x half> %a, <16 x half> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+ %dup_odd_b = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+ %mul_im = fmul contract <16 x half> %dup_odd_b, %swap_a
+ %add = fadd contract <16 x half> %mul_re, %mul_im
+ %sub = fsub contract <16 x half> %mul_re, %mul_im
+ %cmul = shufflevector <16 x half> %add, <16 x half> %sub, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
+ %res = fadd contract <16 x half> %cmul, %acc
+ ret <16 x half> %res
}
-define <32 x half> @test_512(<32 x half> %0, <32 x half> %1, <32 x half> %2) {
+define <32 x half> @test_512(<32 x half> %a, <32 x half> %b, <32 x half> %acc) {
; CHECK-LABEL: test_512:
-; CHECK: # %bb.0: # %entry
+; CHECK: # %bb.0:
; CHECK-NEXT: vfcmaddcph %zmm1, %zmm0, %zmm2
; CHECK-NEXT: vmovaps %zmm2, %zmm0
; CHECK-NEXT: retq
-entry:
- %3 = shufflevector <32 x half> %1, <32 x half> poison, <32 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14, i32 16, i32 16, i32 18, i32 18, i32 20, i32 20, i32 22, i32 22, i32 24, i32 24, i32 26, i32 26, i32 28, i32 28, i32 30, i32 30>
- %4 = fmul fast <32 x half> %3, %0
- %5 = shufflevector <32 x half> %0, <32 x half> poison, <32 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30>
- %6 = shufflevector <32 x half> %1, <32 x half> poison, <32 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15, i32 17, i32 17, i32 19, i32 19, i32 21, i32 21, i32 23, i32 23, i32 25, i32 25, i32 27, i32 27, i32 29, i32 29, i32 31, i32 31>
- %7 = fmul fast <32 x half> %5, %6
- %8 = fadd fast <32 x half> %4, %7
- %9 = fsub fast <32 x half> %4, %7
- %10 = shufflevector <32 x half> %8, <32 x half> %9, <32 x i32> <i32 0, i32 33, i32 2, i32 35, i32 4, i32 37, i32 6, i32 39, i32 8, i32 41, i32 10, i32 43, i32 12, i32 45, i32 14, i32 47, i32 16, i32 49, i32 18, i32 51, i32 20, i32 53, i32 22, i32 55, i32 24, i32 57, i32 26, i32 59, i32 28, i32 61, i32 30, i32 63>
- %11 = fadd fast <32 x half> %2, %10
- ret <32 x half> %11
+ %dup_even_b = shufflevector <32 x half> %b, <32 x half> poison, <32 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14, i32 16, i32 16, i32 18, i32 18, i32 20, i32 20, i32 22, i32 22, i32 24, i32 24, i32 26, i32 26, i32 28, i32 28, i32 30, i32 30>
+ %mul_re = fmul contract <32 x half> %dup_even_b, %a
+ %swap_a = shufflevector <32 x half> %a, <32 x half> poison, <32 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30>
+ %dup_odd_b = shufflevector <32 x half> %b, <32 x half> poison, <32 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15, i32 17, i32 17, i32 19, i32 19, i32 21, i32 21, i32 23, i32 23, i32 25, i32 25, i32 27, i32 27, i32 29, i32 29, i32 31, i32 31>
+ %mul_im = fmul contract <32 x half> %swap_a, %dup_odd_b
+ %add = fadd contract <32 x half> %mul_re, %mul_im
+ %sub = fsub contract <32 x half> %mul_re, %mul_im
+ %cmul = shufflevector <32 x half> %add, <32 x half> %sub, <32 x i32> <i32 0, i32 33, i32 2, i32 35, i32 4, i32 37, i32 6, i32 39, i32 8, i32 41, i32 10, i32 43, i32 12, i32 45, i32 14, i32 47, i32 16, i32 49, i32 18, i32 51, i32 20, i32 53, i32 22, i32 55, i32 24, i32 57, i32 26, i32 59, i32 28, i32 61, i32 30, i32 63>
+ %res = fadd contract <32 x half> %acc, %cmul
+ ret <32 x half> %res
}
>From 6163775ced97c8f5eeaaa82c4e109b102b7727f0 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Sat, 12 Sep 2026 21:10:12 +0530
Subject: [PATCH 5/6] format
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9259f0475d784..03403d3469733 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56082,15 +56082,16 @@ static bool isCFMulFromFMSUBADD(SDValue N, SelectionDAG &DAG, SDValue &A,
SDValue Op2 = N.getOperand(2);
SmallVector<SDValue, 2> Inputs;
- auto matchShufflePattern = [&DAG, &Inputs](SDValue V, ArrayRef<int> Pat) -> bool {
+ auto matchShufflePattern = [&DAG, &Inputs](SDValue V,
+ ArrayRef<int> Pat) -> bool {
SmallVector<int, 32> Mask;
SmallVector<int, 8> RepeatedMask;
Inputs.clear();
if (!getTargetShuffleInputs(V, Inputs, Mask, DAG))
return false;
MVT VT = V.getSimpleValueType();
- return is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask)
- && isShuffleEquivalent(RepeatedMask, Pat, Inputs[0]);
+ return is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask) &&
+ isShuffleEquivalent(RepeatedMask, Pat, Inputs[0]);
};
auto matchFMSUBADDPattern = [&](SDValue X, SDValue OpA) -> bool {
@@ -56103,10 +56104,9 @@ static bool isCFMulFromFMSUBADD(SDValue N, SelectionDAG &DAG, SDValue &A,
SDValue P = Op2.getOperand(0);
SDValue Q = Op2.getOperand(1);
auto matchFMulPattern = [&](SDValue P, SDValue Q) {
- return matchShufflePattern(P, {1, 0, 3, 2, 5, 4, 7, 6})
- && Inputs[0] == A
- && matchShufflePattern(Q, {1, 1, 3, 3, 5, 5, 7, 7})
- && Inputs[0] == B;
+ return matchShufflePattern(P, {1, 0, 3, 2, 5, 4, 7, 6}) &&
+ Inputs[0] == A &&
+ matchShufflePattern(Q, {1, 1, 3, 3, 5, 5, 7, 7}) && Inputs[0] == B;
};
return matchFMulPattern(P, Q) || matchFMulPattern(Q, P);
};
>From 368422f55f6b0abc5436244743709b1c02cadef7 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Mon, 14 Sep 2026 16:12:10 +0530
Subject: [PATCH 6/6] cleanup
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 32 +++++++++++--------------
1 file changed, 14 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 03403d3469733..e4167291548bf 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56080,38 +56080,34 @@ static bool isCFMulFromFMSUBADD(SDValue N, SelectionDAG &DAG, SDValue &A,
SDValue Op0 = N.getOperand(0);
SDValue Op1 = N.getOperand(1);
SDValue Op2 = N.getOperand(2);
- SmallVector<SDValue, 2> Inputs;
- auto matchShufflePattern = [&DAG, &Inputs](SDValue V,
- ArrayRef<int> Pat) -> bool {
+ auto matchShufflePattern = [&DAG](SDValue V, ArrayRef<int> Pat) {
+ SmallVector<SDValue, 2> Inputs;
SmallVector<int, 32> Mask;
SmallVector<int, 8> RepeatedMask;
- Inputs.clear();
- if (!getTargetShuffleInputs(V, Inputs, Mask, DAG))
- return false;
MVT VT = V.getSimpleValueType();
- return is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask) &&
- isShuffleEquivalent(RepeatedMask, Pat, Inputs[0]);
+ if (getTargetShuffleInputs(V, Inputs, Mask, DAG) &&
+ is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask) &&
+ isShuffleEquivalent(RepeatedMask, Pat, Inputs[0]))
+ return Inputs[0];
+ return SDValue();
};
-
- auto matchFMSUBADDPattern = [&](SDValue X, SDValue OpA) -> bool {
- if (!matchShufflePattern(X, {0, 0, 2, 2, 4, 4, 6, 6}))
+ auto matchFMSUBADDPattern = [&](SDValue X, SDValue OpA) {
+ B = matchShufflePattern(X, {0, 0, 2, 2, 4, 4, 6, 6});
+ if (!B)
return false;
- B = Inputs[0];
A = OpA;
- if (Op2.getOpcode() != ISD::FMUL)
- return false;
SDValue P = Op2.getOperand(0);
SDValue Q = Op2.getOperand(1);
auto matchFMulPattern = [&](SDValue P, SDValue Q) {
- return matchShufflePattern(P, {1, 0, 3, 2, 5, 4, 7, 6}) &&
- Inputs[0] == A &&
- matchShufflePattern(Q, {1, 1, 3, 3, 5, 5, 7, 7}) && Inputs[0] == B;
+ return matchShufflePattern(P, {1, 0, 3, 2, 5, 4, 7, 6}) == A &&
+ matchShufflePattern(Q, {1, 1, 3, 3, 5, 5, 7, 7}) == B;
};
return matchFMulPattern(P, Q) || matchFMulPattern(Q, P);
};
// First 2 operands of FMSUBADD are commutable.
- return matchFMSUBADDPattern(Op0, Op1) || matchFMSUBADDPattern(Op1, Op0);
+ return Op2.getOpcode() == ISD::FMUL &&
+ (matchFMSUBADDPattern(Op0, Op1) || matchFMSUBADDPattern(Op1, Op0));
}
// Try to combine the following nodes:
More information about the llvm-commits
mailing list