[llvm] [ComplexDeinterleaving] Add fma/fmuladd support in identifyReassocNodes (PR #217563)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 02:49:43 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Bryth (matteo-rm)
<details>
<summary>Changes</summary>
In some cases, MLIR can emit `fma`/`fmuladd` intrinsics. Those intrinsics are currently not used when matching for FCMLA patterns, missing potential optimizations. This PR adds a case to handle `fma`/`fmuladd` in the pass and associated tests.
Note: `fma` semantics (only one float rounding) are preserved since `fmul`+`fadd` is always emitted as a fused op. Reordering is permitted under the `reassoc` FMF required here.
---
Full diff: https://github.com/llvm/llvm-project/pull/217563.diff
3 Files Affected:
- (modified) llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp (+32-3)
- (added) llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll (+38)
- (added) llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll (+29)
``````````diff
diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
index 60a660557a0f3..e7c38dd7608a6 100644
--- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
+++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
@@ -1212,14 +1212,19 @@ ComplexDeinterleavingGraph::identifyNode(ComplexValues &Vals) {
ComplexDeinterleavingGraph::CompositeNode *
ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
Instruction *Imag) {
- auto IsOperationSupported = [](unsigned Opcode) -> bool {
+ auto IsOperationSupported = [](Instruction *I) -> bool {
+ unsigned Opcode = I->getOpcode();
+ if (Opcode == Instruction::Call)
+ return match(I, m_Intrinsic<Intrinsic::fma>(m_Value(), m_Value(),
+ m_Value())) ||
+ match(I, m_Intrinsic<Intrinsic::fmuladd>(m_Value(), m_Value(),
+ m_Value()));
return Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
Opcode == Instruction::FNeg || Opcode == Instruction::Add ||
Opcode == Instruction::Sub;
};
- if (!IsOperationSupported(Real->getOpcode()) ||
- !IsOperationSupported(Imag->getOpcode()))
+ if (!IsOperationSupported(Real) || !IsOperationSupported(Imag))
return nullptr;
std::optional<FastMathFlags> Flags;
@@ -1305,6 +1310,30 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
case Instruction::FNeg:
Worklist.emplace_back(I->getOperand(0), !IsPositive);
break;
+ case Instruction::Call: {
+ Value *A, *B, *C;
+ if (!match(I, m_Intrinsic<Intrinsic::fma>(m_Value(A), m_Value(B),
+ m_Value(C))) &&
+ !match(I, m_Intrinsic<Intrinsic::fmuladd>(m_Value(A), m_Value(B),
+ m_Value(C)))) {
+ Addends.emplace_back(I, IsPositive);
+ continue;
+ }
+
+ if (isNeg(A)) {
+ A = getNegOperand(A);
+ IsPositive = !IsPositive;
+ }
+
+ if (isNeg(B)) {
+ B = getNegOperand(B);
+ IsPositive = !IsPositive;
+ }
+
+ Muls.push_back(Product{A, B, IsPositive});
+ Worklist.emplace_back(C, IsPositive);
+ break;
+ }
default:
Addends.emplace_back(I, IsPositive);
continue;
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll
new file mode 100644
index 0000000000000..515c082abe01a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+sve -o - | FileCheck %s
+
+target triple = "aarch64"
+
+; Expected to transform
+define <vscale x 8 x float> @complex_mul_fma_scalable(<vscale x 8 x float> %a, <vscale x 8 x float> %b) {
+; CHECK-LABEL: complex_mul_fma_scalable:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmla z5.s, p0/m, z0.s, z2.s, #0
+; CHECK-NEXT: fcmla z4.s, p0/m, z1.s, z3.s, #0
+; CHECK-NEXT: fcmla z5.s, p0/m, z0.s, z2.s, #90
+; CHECK-NEXT: fcmla z4.s, p0/m, z1.s, z3.s, #90
+; CHECK-NEXT: mov z0.d, z5.d
+; CHECK-NEXT: mov z1.d, z4.d
+; CHECK-NEXT: ret
+entry:
+ %da = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %a)
+ %a.re = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %da, 0
+ %a.im = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %da, 1
+ %db = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %b)
+ %b.re = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %db, 0
+ %b.im = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %db, 1
+ %neg = fneg fast <vscale x 4 x float> %a.im
+ %t0 = fmul fast <vscale x 4 x float> %b.im, %neg
+ %re = tail call fast <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %a.re, <vscale x 4 x float> %b.re, <vscale x 4 x float> %t0)
+ %t1 = fmul fast <vscale x 4 x float> %b.re, %a.im
+ %im = tail call fast <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %a.re, <vscale x 4 x float> %b.im, <vscale x 4 x float> %t1)
+ %r = tail call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %re, <vscale x 4 x float> %im)
+ ret <vscale x 8 x float> %r
+}
+
+declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float>)
+declare <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
new file mode 100644
index 0000000000000..5022db15f2a82
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+complxnum,+neon -o - | FileCheck %s
+
+target triple = "aarch64"
+
+; Expected to transform
+define <4 x float> @complex_mul_fmuladd(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: complex_mul_fmuladd:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: fcmla v2.4s, v0.4s, v1.4s, #0
+; CHECK-NEXT: fcmla v2.4s, v0.4s, v1.4s, #90
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %a.re = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %a.im = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %b.re = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %b.im = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+
+ %t0 = fmul fast <2 x float> %a.im, %b.im
+ %neg = fneg fast <2 x float> %t0
+ %re = call fast <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a.re, <2 x float> %b.re, <2 x float> %neg)
+ %t1 = fmul fast <2 x float> %a.im, %b.re
+ %im = call fast <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a.re, <2 x float> %b.im, <2 x float> %t1)
+ %r = shufflevector <2 x float> %re, <2 x float> %im, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+ ret <4 x float> %r
+}
+
+declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>)
``````````
</details>
https://github.com/llvm/llvm-project/pull/217563
More information about the llvm-commits
mailing list