[llvm] [X86] Fold FMADDSUB into VFMULC for fp16 complex multiply (PR #227698)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 06:08:43 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: Rohan Shenoy (222rohan)
<details>
<summary>Changes</summary>
We fold fmaddsub using isCFMulFromFMAddSub (formerly isCFMulFromFMSUBADD) into vfmulc.
---
Full diff: https://github.com/llvm/llvm-project/pull/227698.diff
3 Files Affected:
- (modified) llvm/lib/Target/X86/X86ISelLowering.cpp (+15-11)
- (modified) llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll (+54)
- (modified) llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd.ll (+51)
``````````diff
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7817c86155e5b..eb55eeda49671 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56168,8 +56168,8 @@ static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG,
return Res;
}
-// We try to match the following pattern from FMSUBADD(X, A, M) to lower it
-// into complex conjugate multiply for fp16.
+// We try to match the following pattern from FMADDSUB/FMSUBADD(X, A, M) to
+// lower it into complex multiply for fp16.
// for vector of the complex form v <v0r, v0i, v1r, v1i, ...>
// and 2 complex vectors a, b,
// X = duplicate real (b) : <b0r, b0r, b1r, b1r, ...>
@@ -56177,7 +56177,7 @@ static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG,
// M = FMUL (P, Q)
// P = adjacent pair swapped (a) : <a0i, a0r, a1i, a1r, ...>
// Q = duplicate imaginary (b) : <b0i, b0i, b1i, b1i, ...>
-static bool isCFMulFromFMSUBADD(SDValue N, SelectionDAG &DAG, SDValue &A,
+static bool isCFMulFromFMAddSub(SDValue N, SelectionDAG &DAG, SDValue &A,
SDValue &B) {
SDValue Op0 = N.getOperand(0);
SDValue Op1 = N.getOperand(1);
@@ -56207,7 +56207,7 @@ static bool isCFMulFromFMSUBADD(SDValue N, SelectionDAG &DAG, SDValue &A,
};
return matchFMulPattern(P, Q) || matchFMulPattern(Q, P);
};
- // First 2 operands of FMSUBADD are commutable.
+ // First 2 operands of FMADDSUB/FMSUBADD are commutable.
return Op2.getOpcode() == ISD::FMUL &&
(matchFMSUBADDPattern(Op0, Op1) || matchFMSUBADDPattern(Op1, Op0));
}
@@ -58502,6 +58502,7 @@ static SDValue combineFMA(SDNode *N, SelectionDAG &DAG,
}
}
+// Combine FMADDSUB(SHUFFLE(B),A,FMUL(SHUFFLE(A),SHUFFLE(B))) -> VFMULC(A,B)
// Combine FMSUBADD(SHUFFLE(B),A,FMUL(SHUFFLE(A),SHUFFLE(B))) -> VFCMULC(A,B)
// Combine FMADDSUB(A, B, FNEG(C)) -> FMSUBADD(A, B, C)
// Combine FMSUBADD(A, B, FNEG(C)) -> FMADDSUB(A, B, C)
@@ -58512,19 +58513,22 @@ static SDValue combineFMADDSUB(SDNode *N, SelectionDAG &DAG,
EVT VT = N->getValueType(0);
SDValue N2 = N->getOperand(2);
- if (N->getOpcode() == X86ISD::FMSUBADD && Subtarget.hasFP16() &&
- N->hasOneUse() &&
+ unsigned Opc = N->getOpcode();
+ if ((Opc == X86ISD::FMADDSUB || Opc == X86ISD::FMSUBADD) &&
+ Subtarget.hasFP16() && N->hasOneUse() &&
(VT == MVT::v8f16 || VT == MVT::v16f16 || VT == MVT::v32f16)) {
SDValue A, B;
- if (isCFMulFromFMSUBADD(SDValue(N, 0), DAG, A, B)) {
+ if (isCFMulFromFMAddSub(SDValue(N, 0), DAG, A, B)) {
MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2);
SDValue MulOp0 = DAG.getBitcast(CVT, A);
SDValue MulOp1 = DAG.getBitcast(CVT, B);
- // FMSUBADD has no flags, so we use the flags from the FMUL (i.e. the
- // third operand) it was fused from, as it is the only operand which
- // still has FMF (see isCFMulFromFMSUBADD for the pattern).
+ // FMADDSUB/FMSUBADD has no flags, so we use the flags from the FMUL
+ // (i.e. the third operand) it was fused from, as it is the only operand
+ // which still has FMF (see isCFMulFromFMAddSub for the pattern).
+ unsigned NewOpc =
+ Opc == X86ISD::FMADDSUB ? X86ISD::VFMULC : X86ISD::VFCMULC;
SDValue Fmulc =
- DAG.getNode(X86ISD::VFCMULC, dl, CVT, MulOp0, MulOp1, N2->getFlags());
+ DAG.getNode(NewOpc, dl, CVT, MulOp0, MulOp1, N2->getFlags());
return DAG.getBitcast(VT, Fmulc);
}
}
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
index 4d1e7479d0039..b9268c344d8e7 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd-fadd.ll
@@ -54,3 +54,57 @@ define <32 x half> @test_512(<32 x half> %a, <32 x half> %b, <32 x half> %acc) {
%res = fadd contract <32 x half> %acc, %cmul
ret <32 x half> %res
}
+
+define <8 x half> @test_mulc_128(<8 x half> %a, <8 x half> %b, <8 x half> %acc) {
+; CHECK-LABEL: test_mulc_128:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vfmaddcph %xmm1, %xmm0, %xmm2
+; CHECK-NEXT: vmovaps %xmm2, %xmm0
+; CHECK-NEXT: retq
+ %dup_even_b = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+ %mul_re = fmul contract <8 x half> %a, %dup_even_b
+ %swap_a = shufflevector <8 x half> %a, <8 x half> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+ %dup_odd_b = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+ %mul_im = fmul contract <8 x half> %swap_a, %dup_odd_b
+ %add = fadd contract <8 x half> %mul_re, %mul_im
+ %sub = fsub contract <8 x half> %mul_re, %mul_im
+ %cmul = shufflevector <8 x half> %sub, <8 x half> %add, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ %res = fadd contract <8 x half> %cmul, %acc
+ ret <8 x half> %res
+}
+
+define <16 x half> @test_mulc_256(<16 x half> %a, <16 x half> %b, <16 x half> %acc) {
+; CHECK-LABEL: test_mulc_256:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vfmaddcph %ymm1, %ymm0, %ymm2
+; CHECK-NEXT: vmovaps %ymm2, %ymm0
+; CHECK-NEXT: retq
+ %dup_even_b = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+ %mul_re = fmul contract <16 x half> %dup_even_b, %a
+ %swap_a = shufflevector <16 x half> %a, <16 x half> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+ %dup_odd_b = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+ %mul_im = fmul contract <16 x half> %dup_odd_b, %swap_a
+ %add = fadd contract <16 x half> %mul_re, %mul_im
+ %sub = fsub contract <16 x half> %mul_re, %mul_im
+ %cmul = shufflevector <16 x half> %sub, <16 x half> %add, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
+ %res = fadd contract <16 x half> %cmul, %acc
+ ret <16 x half> %res
+}
+
+define <32 x half> @test_mulc_512(<32 x half> %a, <32 x half> %b, <32 x half> %acc) {
+; CHECK-LABEL: test_mulc_512:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vfmaddcph %zmm1, %zmm0, %zmm2
+; CHECK-NEXT: vmovaps %zmm2, %zmm0
+; CHECK-NEXT: retq
+ %dup_even_b = shufflevector <32 x half> %b, <32 x half> poison, <32 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14, i32 16, i32 16, i32 18, i32 18, i32 20, i32 20, i32 22, i32 22, i32 24, i32 24, i32 26, i32 26, i32 28, i32 28, i32 30, i32 30>
+ %mul_re = fmul contract <32 x half> %dup_even_b, %a
+ %swap_a = shufflevector <32 x half> %a, <32 x half> poison, <32 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30>
+ %dup_odd_b = shufflevector <32 x half> %b, <32 x half> poison, <32 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15, i32 17, i32 17, i32 19, i32 19, i32 21, i32 21, i32 23, i32 23, i32 25, i32 25, i32 27, i32 27, i32 29, i32 29, i32 31, i32 31>
+ %mul_im = fmul contract <32 x half> %swap_a, %dup_odd_b
+ %add = fadd contract <32 x half> %mul_re, %mul_im
+ %sub = fsub contract <32 x half> %mul_re, %mul_im
+ %cmul = shufflevector <32 x half> %sub, <32 x half> %add, <32 x i32> <i32 0, i32 33, i32 2, i32 35, i32 4, i32 37, i32 6, i32 39, i32 8, i32 41, i32 10, i32 43, i32 12, i32 45, i32 14, i32 47, i32 16, i32 49, i32 18, i32 51, i32 20, i32 53, i32 22, i32 55, i32 24, i32 57, i32 26, i32 59, i32 28, i32 61, i32 30, i32 63>
+ %res = fadd contract <32 x half> %acc, %cmul
+ ret <32 x half> %res
+}
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd.ll
index c2ff3c51b5705..d77af74c7de41 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-fmsubadd.ll
@@ -51,3 +51,54 @@ define <32 x half> @test_512(<32 x half> %a, <32 x half> %b) {
%cmul = shufflevector <32 x half> %add, <32 x half> %sub, <32 x i32> <i32 0, i32 33, i32 2, i32 35, i32 4, i32 37, i32 6, i32 39, i32 8, i32 41, i32 10, i32 43, i32 12, i32 45, i32 14, i32 47, i32 16, i32 49, i32 18, i32 51, i32 20, i32 53, i32 22, i32 55, i32 24, i32 57, i32 26, i32 59, i32 28, i32 61, i32 30, i32 63>
ret <32 x half> %cmul
}
+
+define <8 x half> @test_mulc_128(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: test_mulc_128:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vfmulcph %xmm1, %xmm0, %xmm2
+; CHECK-NEXT: vmovaps %xmm2, %xmm0
+; CHECK-NEXT: retq
+ %dup_even_b = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+ %mul_re = fmul contract <8 x half> %a, %dup_even_b
+ %swap_a = shufflevector <8 x half> %a, <8 x half> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+ %dup_odd_b = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+ %mul_im = fmul contract <8 x half> %swap_a, %dup_odd_b
+ %add = fadd contract <8 x half> %mul_re, %mul_im
+ %sub = fsub contract <8 x half> %mul_re, %mul_im
+ %cmul = shufflevector <8 x half> %sub, <8 x half> %add, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+ ret <8 x half> %cmul
+}
+
+define <16 x half> @test_mulc_256(<16 x half> %a, <16 x half> %b) {
+; CHECK-LABEL: test_mulc_256:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vfmulcph %ymm1, %ymm0, %ymm2
+; CHECK-NEXT: vmovaps %ymm2, %ymm0
+; CHECK-NEXT: retq
+ %dup_even_b = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+ %mul_re = fmul contract <16 x half> %dup_even_b, %a
+ %swap_a = shufflevector <16 x half> %a, <16 x half> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+ %dup_odd_b = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+ %mul_im = fmul contract <16 x half> %dup_odd_b, %swap_a
+ %add = fadd contract <16 x half> %mul_re, %mul_im
+ %sub = fsub contract <16 x half> %mul_re, %mul_im
+ %cmul = shufflevector <16 x half> %sub, <16 x half> %add, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
+ ret <16 x half> %cmul
+}
+
+define <32 x half> @test_mulc_512(<32 x half> %a, <32 x half> %b) {
+; CHECK-LABEL: test_mulc_512:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vfmulcph %zmm1, %zmm0, %zmm2
+; CHECK-NEXT: vmovaps %zmm2, %zmm0
+; CHECK-NEXT: retq
+ %dup_even_b = shufflevector <32 x half> %b, <32 x half> poison, <32 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14, i32 16, i32 16, i32 18, i32 18, i32 20, i32 20, i32 22, i32 22, i32 24, i32 24, i32 26, i32 26, i32 28, i32 28, i32 30, i32 30>
+ %mul_re = fmul contract <32 x half> %dup_even_b, %a
+ %swap_a = shufflevector <32 x half> %a, <32 x half> poison, <32 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30>
+ %dup_odd_b = shufflevector <32 x half> %b, <32 x half> poison, <32 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15, i32 17, i32 17, i32 19, i32 19, i32 21, i32 21, i32 23, i32 23, i32 25, i32 25, i32 27, i32 27, i32 29, i32 29, i32 31, i32 31>
+ %mul_im = fmul contract <32 x half> %swap_a, %dup_odd_b
+ %add = fadd contract <32 x half> %mul_re, %mul_im
+ %sub = fsub contract <32 x half> %mul_re, %mul_im
+ %cmul = shufflevector <32 x half> %sub, <32 x half> %add, <32 x i32> <i32 0, i32 33, i32 2, i32 35, i32 4, i32 37, i32 6, i32 39, i32 8, i32 41, i32 10, i32 43, i32 12, i32 45, i32 14, i32 47, i32 16, i32 49, i32 18, i32 51, i32 20, i32 53, i32 22, i32 55, i32 24, i32 57, i32 26, i32 59, i32 28, i32 61, i32 30, i32 63>
+ ret <32 x half> %cmul
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/227698
More information about the llvm-commits
mailing list