[llvm] [ComplexDeinterleaving] Add fma/fmuladd support in identifyReassocNodes (PR #217563)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 07:48:45 PDT 2026
=?utf-8?q?Mattéo?= Rizza Murgier,=?utf-8?q?Mattéo?= Rizza Murgier
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217563 at github.com>
https://github.com/matteo-rm updated https://github.com/llvm/llvm-project/pull/217563
>From 746584a0ce04adc52a3294032dc7c31517004810 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matt=C3=A9o=20Rizza=20Murgier?=
<matteo.rizza-murgier at sipearl.com>
Date: Wed, 19 Aug 2026 14:20:38 +0200
Subject: [PATCH 1/3] [ComplexDeinterleaving] Add fma/fmuladd support in
identifyReassocNodes
---
.../lib/CodeGen/ComplexDeinterleavingPass.cpp | 35 +++++++++++++++--
.../complex-deinterleaving-fma-scalable.ll | 38 +++++++++++++++++++
.../AArch64/complex-deinterleaving-fma.ll | 29 ++++++++++++++
3 files changed, 99 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll
create mode 100644 llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
index 60a660557a0f3..e7c38dd7608a6 100644
--- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
+++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
@@ -1212,14 +1212,19 @@ ComplexDeinterleavingGraph::identifyNode(ComplexValues &Vals) {
ComplexDeinterleavingGraph::CompositeNode *
ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
Instruction *Imag) {
- auto IsOperationSupported = [](unsigned Opcode) -> bool {
+ auto IsOperationSupported = [](Instruction *I) -> bool {
+ unsigned Opcode = I->getOpcode();
+ if (Opcode == Instruction::Call)
+ return match(I, m_Intrinsic<Intrinsic::fma>(m_Value(), m_Value(),
+ m_Value())) ||
+ match(I, m_Intrinsic<Intrinsic::fmuladd>(m_Value(), m_Value(),
+ m_Value()));
return Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
Opcode == Instruction::FNeg || Opcode == Instruction::Add ||
Opcode == Instruction::Sub;
};
- if (!IsOperationSupported(Real->getOpcode()) ||
- !IsOperationSupported(Imag->getOpcode()))
+ if (!IsOperationSupported(Real) || !IsOperationSupported(Imag))
return nullptr;
std::optional<FastMathFlags> Flags;
@@ -1305,6 +1310,30 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
case Instruction::FNeg:
Worklist.emplace_back(I->getOperand(0), !IsPositive);
break;
+ case Instruction::Call: {
+ Value *A, *B, *C;
+ if (!match(I, m_Intrinsic<Intrinsic::fma>(m_Value(A), m_Value(B),
+ m_Value(C))) &&
+ !match(I, m_Intrinsic<Intrinsic::fmuladd>(m_Value(A), m_Value(B),
+ m_Value(C)))) {
+ Addends.emplace_back(I, IsPositive);
+ continue;
+ }
+
+ if (isNeg(A)) {
+ A = getNegOperand(A);
+ IsPositive = !IsPositive;
+ }
+
+ if (isNeg(B)) {
+ B = getNegOperand(B);
+ IsPositive = !IsPositive;
+ }
+
+ Muls.push_back(Product{A, B, IsPositive});
+ Worklist.emplace_back(C, IsPositive);
+ break;
+ }
default:
Addends.emplace_back(I, IsPositive);
continue;
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll
new file mode 100644
index 0000000000000..515c082abe01a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+sve -o - | FileCheck %s
+
+target triple = "aarch64"
+
+; Expected to transform
+define <vscale x 8 x float> @complex_mul_fma_scalable(<vscale x 8 x float> %a, <vscale x 8 x float> %b) {
+; CHECK-LABEL: complex_mul_fma_scalable:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmla z5.s, p0/m, z0.s, z2.s, #0
+; CHECK-NEXT: fcmla z4.s, p0/m, z1.s, z3.s, #0
+; CHECK-NEXT: fcmla z5.s, p0/m, z0.s, z2.s, #90
+; CHECK-NEXT: fcmla z4.s, p0/m, z1.s, z3.s, #90
+; CHECK-NEXT: mov z0.d, z5.d
+; CHECK-NEXT: mov z1.d, z4.d
+; CHECK-NEXT: ret
+entry:
+ %da = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %a)
+ %a.re = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %da, 0
+ %a.im = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %da, 1
+ %db = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %b)
+ %b.re = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %db, 0
+ %b.im = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %db, 1
+ %neg = fneg fast <vscale x 4 x float> %a.im
+ %t0 = fmul fast <vscale x 4 x float> %b.im, %neg
+ %re = tail call fast <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %a.re, <vscale x 4 x float> %b.re, <vscale x 4 x float> %t0)
+ %t1 = fmul fast <vscale x 4 x float> %b.re, %a.im
+ %im = tail call fast <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %a.re, <vscale x 4 x float> %b.im, <vscale x 4 x float> %t1)
+ %r = tail call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %re, <vscale x 4 x float> %im)
+ ret <vscale x 8 x float> %r
+}
+
+declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float>)
+declare <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
new file mode 100644
index 0000000000000..5022db15f2a82
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+complxnum,+neon -o - | FileCheck %s
+
+target triple = "aarch64"
+
+; Expected to transform
+define <4 x float> @complex_mul_fmuladd(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: complex_mul_fmuladd:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: fcmla v2.4s, v0.4s, v1.4s, #0
+; CHECK-NEXT: fcmla v2.4s, v0.4s, v1.4s, #90
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %a.re = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %a.im = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %b.re = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %b.im = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+
+ %t0 = fmul fast <2 x float> %a.im, %b.im
+ %neg = fneg fast <2 x float> %t0
+ %re = call fast <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a.re, <2 x float> %b.re, <2 x float> %neg)
+ %t1 = fmul fast <2 x float> %a.im, %b.re
+ %im = call fast <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a.re, <2 x float> %b.im, <2 x float> %t1)
+ %r = shufflevector <2 x float> %re, <2 x float> %im, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+ ret <4 x float> %r
+}
+
+declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>)
>From 454b9d1d65d84d5f8a9ea46dcc6c7cf8b015ca57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matt=C3=A9o=20Rizza=20Murgier?=
<matteo.rizza-murgier at sipearl.com>
Date: Thu, 20 Aug 2026 16:46:17 +0200
Subject: [PATCH 2/3] [ComplexDeinterleaving] simplify IsOperationSupported
check
---
llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
index e7c38dd7608a6..513c7782630ac 100644
--- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
+++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
@@ -1214,12 +1214,8 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
Instruction *Imag) {
auto IsOperationSupported = [](Instruction *I) -> bool {
unsigned Opcode = I->getOpcode();
- if (Opcode == Instruction::Call)
- return match(I, m_Intrinsic<Intrinsic::fma>(m_Value(), m_Value(),
- m_Value())) ||
- match(I, m_Intrinsic<Intrinsic::fmuladd>(m_Value(), m_Value(),
- m_Value()));
- return Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
+ return match(I, m_AnyIntrinsic<Intrinsic::fma, Intrinsic::fmuladd>()) ||
+ Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
Opcode == Instruction::FNeg || Opcode == Instruction::Add ||
Opcode == Instruction::Sub;
};
>From 889bd335d94def759a7928b1cca6b9339bd58302 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matt=C3=A9o=20Rizza=20Murgier?=
<matteo.rizza-murgier at sipearl.com>
Date: Thu, 20 Aug 2026 16:47:08 +0200
Subject: [PATCH 3/3] [ComplexDeinterleaving] add negative tests
---
.../complex-deinterleaving-fma-scalable.ll | 34 +++++++++++++++++++
.../AArch64/complex-deinterleaving-fma.ll | 31 +++++++++++++++++
2 files changed, 65 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll
index 515c082abe01a..168c1bee90ba5 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll
@@ -33,6 +33,40 @@ entry:
ret <vscale x 8 x float> %r
}
+; Expected not to transform
+define <vscale x 8 x float> @complex_mul_fma_scalable_no_fmf(<vscale x 8 x float> %a, <vscale x 8 x float> %b) {
+; CHECK-LABEL: complex_mul_fma_scalable_no_fmf:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzp2 z4.s, z0.s, z1.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: uzp1 z6.s, z2.s, z3.s
+; CHECK-NEXT: uzp2 z2.s, z2.s, z3.s
+; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s
+; CHECK-NEXT: movprfx z5, z4
+; CHECK-NEXT: fneg z5.s, p0/m, z4.s
+; CHECK-NEXT: fmul z4.s, z6.s, z4.s
+; CHECK-NEXT: fmul z3.s, z2.s, z5.s
+; CHECK-NEXT: fmad z2.s, p0/m, z0.s, z4.s
+; CHECK-NEXT: fmla z3.s, p0/m, z0.s, z6.s
+; CHECK-NEXT: zip1 z0.s, z3.s, z2.s
+; CHECK-NEXT: zip2 z1.s, z3.s, z2.s
+; CHECK-NEXT: ret
+entry:
+ %da = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %a)
+ %a.re = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %da, 0
+ %a.im = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %da, 1
+ %db = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %b)
+ %b.re = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %db, 0
+ %b.im = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %db, 1
+ %neg = fneg <vscale x 4 x float> %a.im
+ %t0 = fmul <vscale x 4 x float> %b.im, %neg
+ %re = tail call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %a.re, <vscale x 4 x float> %b.re, <vscale x 4 x float> %t0)
+ %t1 = fmul <vscale x 4 x float> %b.re, %a.im
+ %im = tail call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %a.re, <vscale x 4 x float> %b.im, <vscale x 4 x float> %t1)
+ %r = tail call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %re, <vscale x 4 x float> %im)
+ ret <vscale x 8 x float> %r
+}
+
declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float>)
declare <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float>, <vscale x 4 x float>)
declare <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
index 5022db15f2a82..5970b2bb2dfd7 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
@@ -26,4 +26,35 @@ define <4 x float> @complex_mul_fmuladd(<4 x float> %a, <4 x float> %b) {
ret <4 x float> %r
}
+; Expected not to transform
+define <4 x float> @complex_mul_fmuladd_no_fmf(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: complex_mul_fmuladd_no_fmf:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov d2, v0.d[1]
+; CHECK-NEXT: mov d3, v1.d[1]
+; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
+; CHECK-NEXT: zip2 v5.2s, v1.2s, v3.2s
+; CHECK-NEXT: zip1 v1.2s, v1.2s, v3.2s
+; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
+; CHECK-NEXT: fmul v3.2s, v4.2s, v5.2s
+; CHECK-NEXT: fneg v2.2s, v3.2s
+; CHECK-NEXT: fmul v3.2s, v4.2s, v1.2s
+; CHECK-NEXT: fmla v2.2s, v1.2s, v0.2s
+; CHECK-NEXT: fmla v3.2s, v5.2s, v0.2s
+; CHECK-NEXT: zip1 v0.4s, v2.4s, v3.4s
+; CHECK-NEXT: ret
+ %a.re = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %a.im = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %b.re = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %b.im = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+
+ %t0 = fmul <2 x float> %a.im, %b.im
+ %neg = fneg <2 x float> %t0
+ %re = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a.re, <2 x float> %b.re, <2 x float> %neg)
+ %t1 = fmul <2 x float> %a.im, %b.re
+ %im = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a.re, <2 x float> %b.im, <2 x float> %t1)
+ %r = shufflevector <2 x float> %re, <2 x float> %im, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+ ret <4 x float> %r
+}
+
declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>)
More information about the llvm-commits
mailing list