[llvm] e9c9a41 - [ComplexDeinterleaving] Add fma/fmuladd support in identifyReassocNodes (#217563)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 09:53:27 PDT 2026
Author: Bryth
Date: 2026-08-20T18:53:22+02:00
New Revision: e9c9a4185530497aa5a7abb548acf1f3e0340e02
URL: https://github.com/llvm/llvm-project/commit/e9c9a4185530497aa5a7abb548acf1f3e0340e02
DIFF: https://github.com/llvm/llvm-project/commit/e9c9a4185530497aa5a7abb548acf1f3e0340e02.diff
LOG: [ComplexDeinterleaving] Add fma/fmuladd support in identifyReassocNodes (#217563)
Added:
llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll
llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
Modified:
llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
index 60a660557a0f3..513c7782630ac 100644
--- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
+++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
@@ -1212,14 +1212,15 @@ ComplexDeinterleavingGraph::identifyNode(ComplexValues &Vals) {
ComplexDeinterleavingGraph::CompositeNode *
ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
Instruction *Imag) {
- auto IsOperationSupported = [](unsigned Opcode) -> bool {
- return Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
+ auto IsOperationSupported = [](Instruction *I) -> bool {
+ unsigned Opcode = I->getOpcode();
+ return match(I, m_AnyIntrinsic<Intrinsic::fma, Intrinsic::fmuladd>()) ||
+ Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
Opcode == Instruction::FNeg || Opcode == Instruction::Add ||
Opcode == Instruction::Sub;
};
- if (!IsOperationSupported(Real->getOpcode()) ||
- !IsOperationSupported(Imag->getOpcode()))
+ if (!IsOperationSupported(Real) || !IsOperationSupported(Imag))
return nullptr;
std::optional<FastMathFlags> Flags;
@@ -1305,6 +1306,30 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
case Instruction::FNeg:
Worklist.emplace_back(I->getOperand(0), !IsPositive);
break;
+ case Instruction::Call: {
+ Value *A, *B, *C;
+ if (!match(I, m_Intrinsic<Intrinsic::fma>(m_Value(A), m_Value(B),
+ m_Value(C))) &&
+ !match(I, m_Intrinsic<Intrinsic::fmuladd>(m_Value(A), m_Value(B),
+ m_Value(C)))) {
+ Addends.emplace_back(I, IsPositive);
+ continue;
+ }
+
+ if (isNeg(A)) {
+ A = getNegOperand(A);
+ IsPositive = !IsPositive;
+ }
+
+ if (isNeg(B)) {
+ B = getNegOperand(B);
+ IsPositive = !IsPositive;
+ }
+
+ Muls.push_back(Product{A, B, IsPositive});
+ Worklist.emplace_back(C, IsPositive);
+ break;
+ }
default:
Addends.emplace_back(I, IsPositive);
continue;
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll
new file mode 100644
index 0000000000000..168c1bee90ba5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma-scalable.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+sve -o - | FileCheck %s
+
+target triple = "aarch64"
+
+; Expected to transform
+define <vscale x 8 x float> @complex_mul_fma_scalable(<vscale x 8 x float> %a, <vscale x 8 x float> %b) {
+; CHECK-LABEL: complex_mul_fma_scalable:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmla z5.s, p0/m, z0.s, z2.s, #0
+; CHECK-NEXT: fcmla z4.s, p0/m, z1.s, z3.s, #0
+; CHECK-NEXT: fcmla z5.s, p0/m, z0.s, z2.s, #90
+; CHECK-NEXT: fcmla z4.s, p0/m, z1.s, z3.s, #90
+; CHECK-NEXT: mov z0.d, z5.d
+; CHECK-NEXT: mov z1.d, z4.d
+; CHECK-NEXT: ret
+entry:
+ %da = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %a)
+ %a.re = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %da, 0
+ %a.im = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %da, 1
+ %db = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %b)
+ %b.re = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %db, 0
+ %b.im = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %db, 1
+ %neg = fneg fast <vscale x 4 x float> %a.im
+ %t0 = fmul fast <vscale x 4 x float> %b.im, %neg
+ %re = tail call fast <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %a.re, <vscale x 4 x float> %b.re, <vscale x 4 x float> %t0)
+ %t1 = fmul fast <vscale x 4 x float> %b.re, %a.im
+ %im = tail call fast <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %a.re, <vscale x 4 x float> %b.im, <vscale x 4 x float> %t1)
+ %r = tail call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %re, <vscale x 4 x float> %im)
+ ret <vscale x 8 x float> %r
+}
+
+; Expected not to transform
+define <vscale x 8 x float> @complex_mul_fma_scalable_no_fmf(<vscale x 8 x float> %a, <vscale x 8 x float> %b) {
+; CHECK-LABEL: complex_mul_fma_scalable_no_fmf:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzp2 z4.s, z0.s, z1.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: uzp1 z6.s, z2.s, z3.s
+; CHECK-NEXT: uzp2 z2.s, z2.s, z3.s
+; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s
+; CHECK-NEXT: movprfx z5, z4
+; CHECK-NEXT: fneg z5.s, p0/m, z4.s
+; CHECK-NEXT: fmul z4.s, z6.s, z4.s
+; CHECK-NEXT: fmul z3.s, z2.s, z5.s
+; CHECK-NEXT: fmad z2.s, p0/m, z0.s, z4.s
+; CHECK-NEXT: fmla z3.s, p0/m, z0.s, z6.s
+; CHECK-NEXT: zip1 z0.s, z3.s, z2.s
+; CHECK-NEXT: zip2 z1.s, z3.s, z2.s
+; CHECK-NEXT: ret
+entry:
+ %da = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %a)
+ %a.re = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %da, 0
+ %a.im = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %da, 1
+ %db = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %b)
+ %b.re = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %db, 0
+ %b.im = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %db, 1
+ %neg = fneg <vscale x 4 x float> %a.im
+ %t0 = fmul <vscale x 4 x float> %b.im, %neg
+ %re = tail call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %a.re, <vscale x 4 x float> %b.re, <vscale x 4 x float> %t0)
+ %t1 = fmul <vscale x 4 x float> %b.re, %a.im
+ %im = tail call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> %a.re, <vscale x 4 x float> %b.im, <vscale x 4 x float> %t1)
+ %r = tail call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %re, <vscale x 4 x float> %im)
+ ret <vscale x 8 x float> %r
+}
+
+declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float>)
+declare <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
new file mode 100644
index 0000000000000..5970b2bb2dfd7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+complxnum,+neon -o - | FileCheck %s
+
+target triple = "aarch64"
+
+; Expected to transform
+define <4 x float> @complex_mul_fmuladd(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: complex_mul_fmuladd:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: fcmla v2.4s, v0.4s, v1.4s, #0
+; CHECK-NEXT: fcmla v2.4s, v0.4s, v1.4s, #90
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %a.re = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %a.im = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %b.re = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %b.im = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+
+ %t0 = fmul fast <2 x float> %a.im, %b.im
+ %neg = fneg fast <2 x float> %t0
+ %re = call fast <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a.re, <2 x float> %b.re, <2 x float> %neg)
+ %t1 = fmul fast <2 x float> %a.im, %b.re
+ %im = call fast <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a.re, <2 x float> %b.im, <2 x float> %t1)
+ %r = shufflevector <2 x float> %re, <2 x float> %im, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+ ret <4 x float> %r
+}
+
+; Expected not to transform
+define <4 x float> @complex_mul_fmuladd_no_fmf(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: complex_mul_fmuladd_no_fmf:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov d2, v0.d[1]
+; CHECK-NEXT: mov d3, v1.d[1]
+; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
+; CHECK-NEXT: zip2 v5.2s, v1.2s, v3.2s
+; CHECK-NEXT: zip1 v1.2s, v1.2s, v3.2s
+; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
+; CHECK-NEXT: fmul v3.2s, v4.2s, v5.2s
+; CHECK-NEXT: fneg v2.2s, v3.2s
+; CHECK-NEXT: fmul v3.2s, v4.2s, v1.2s
+; CHECK-NEXT: fmla v2.2s, v1.2s, v0.2s
+; CHECK-NEXT: fmla v3.2s, v5.2s, v0.2s
+; CHECK-NEXT: zip1 v0.4s, v2.4s, v3.4s
+; CHECK-NEXT: ret
+ %a.re = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %a.im = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %b.re = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %b.im = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+
+ %t0 = fmul <2 x float> %a.im, %b.im
+ %neg = fneg <2 x float> %t0
+ %re = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a.re, <2 x float> %b.re, <2 x float> %neg)
+ %t1 = fmul <2 x float> %a.im, %b.re
+ %im = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a.re, <2 x float> %b.im, <2 x float> %t1)
+ %r = shufflevector <2 x float> %re, <2 x float> %im, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+ ret <4 x float> %r
+}
+
+declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>)
More information about the llvm-commits
mailing list