[llvm] [AArch64] Generate indexed FCMLAs for dupped LHS complex multiplication (PR #202384)
Hugo Trachino via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 03:10:50 PDT 2026
https://github.com/nujaa updated https://github.com/llvm/llvm-project/pull/202384
>From edfc91025f4642fefed35aee5681700b2b2cbb1f Mon Sep 17 00:00:00 2001
From: Hugo <hugo.trachino at huawei.com>
Date: Tue, 9 Jun 2026 18:10:36 +0800
Subject: [PATCH] [AArch64] Generate indexed FCMLAs for dupped LHS complex
multiplication
---
.../Target/AArch64/AArch64MIPeepholeOpt.cpp | 131 ++++++++++++++
llvm/test/CodeGen/AArch64/neon-vcmla.ll | 165 ++++++++++++++++++
2 files changed, 296 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
index 900f22abcf8b0..c6c962b97dd93 100644
--- a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
@@ -121,6 +121,7 @@ class AArch64MIPeepholeOptImpl {
bool checkMovImmInstr(MachineInstr &MI, MachineInstr *&MovMI,
MachineInstr *&SubregToRegMI);
+ bool isOperandComplexDup(MachineInstr &MI, int operand);
template <typename T>
bool visitADDSUB(unsigned PosOpc, unsigned NegOpc, MachineInstr &MI);
@@ -143,6 +144,7 @@ class AArch64MIPeepholeOptImpl {
bool visitFMOVDr(MachineInstr &MI);
bool visitUBFMXri(MachineInstr &MI);
bool visitCopy(MachineInstr &MI);
+ bool visitFCMLA(MachineInstr &MI);
};
struct AArch64MIPeepholeOptLegacy : public MachineFunctionPass {
@@ -936,6 +938,130 @@ bool AArch64MIPeepholeOptImpl::visitCopy(MachineInstr &MI) {
return true;
}
+bool AArch64MIPeepholeOptImpl::isOperandComplexDup(MachineInstr &MI,
+ int operand) {
+ Register Reg = MI.getOperand(operand).getReg();
+ MachineInstr *Dup = MRI->getVRegDef(Reg);
+ // Only support vector types which duplicates a whole complex.
+ if (MI.getOpcode() == AArch64::FCMLAv8f16 ||
+ MI.getOpcode() == AArch64::FCMLAv4f16)
+ return Dup->getOpcode() == AArch64::DUPv2i32lane ||
+ Dup->getOpcode() == AArch64::DUPv4i32lane;
+ else if (MI.getOpcode() == AArch64::FCMLAv4f32)
+ return Dup->getOpcode() == AArch64::DUPv2i64lane;
+ return false;
+}
+
+// Folds a complex multiplication (2 FCMLAs with orthogonal rotations) with
+// a broadcasted LHS to two orthognal FCMLA_indexed.
+// This optimisation uses complex multiplication commutativity to allow changing
+// the operands order. Permitting FCMLA indexed on LHS.
+//
+// Ex:
+// Dup v1 lane
+// v5 = FCMLA v0, v1, v2 #0
+// res = FCMLA v5, v1, v2 #90
+// into
+// v5 = FCMLA_indexed v0, v2, v1[lane], #0
+// res = FCMLA_indexed v5, v2, v1[lane], #90
+bool AArch64MIPeepholeOptImpl::visitFCMLA(MachineInstr &MI) {
+ MachineFunction *MF = MI.getMF();
+ // LHS must be a dup but not RHS to prevent continuous rewritting.
+ if (!isOperandComplexDup(MI, 2) || isOperandComplexDup(MI, 3)) {
+ return false;
+ }
+
+ // The parent of the accumulator must be another FCMLA with the same operands
+ Register AccReg = MI.getOperand(1).getReg();
+ MachineInstr *ParentFCMLA = MRI->getVRegDef(AccReg);
+ if (!ParentFCMLA) {
+ return false;
+ }
+ if (MI.getOpcode() != ParentFCMLA->getOpcode()) {
+ return false;
+ }
+
+ Register LHSReg = MI.getOperand(2).getReg();
+ Register RHSReg = MI.getOperand(3).getReg();
+ if (LHSReg != ParentFCMLA->getOperand(2).getReg() ||
+ RHSReg != ParentFCMLA->getOperand(3).getReg()) {
+ return false;
+ }
+
+ // Their rotation must be orthogonal to represent a complex multiplication.
+ int FirstRotation = MI.getOperand(4).getImm();
+ int SecondRotation = ParentFCMLA->getOperand(4).getImm();
+ bool Orthogonal = (SecondRotation - FirstRotation) % 2;
+ if (!Orthogonal) {
+ return false;
+ }
+
+ // Only perform this folding if the FCMLAS are on the same block.
+ if (MI.getParent() != ParentFCMLA->getParent()) {
+ return false;
+ }
+
+ unsigned IndexedFCMLAOpc = 0;
+ switch (MI.getOpcode()) {
+ default: {
+ return false;
+ }
+ case AArch64::FCMLAv8f16:
+ IndexedFCMLAOpc = AArch64::FCMLAv8f16_indexed;
+ break;
+ case AArch64::FCMLAv4f16:
+ IndexedFCMLAOpc = AArch64::FCMLAv4f16_indexed;
+ break;
+ case AArch64::FCMLAv4f32:
+ IndexedFCMLAOpc = AArch64::FCMLAv4f32_indexed;
+ break;
+ }
+ MachineInstr *Dup = MF->getRegInfo().getUniqueVRegDef(LHSReg);
+ Register DupSrcReg = Dup->getOperand(1).getReg();
+ unsigned DupSrcLane = Dup->getOperand(2).getImm();
+ MRI->clearKillFlags(DupSrcReg);
+
+ Register DstReg = MI.getOperand(0).getReg();
+ MRI->constrainRegClass(AccReg, MRI->getRegClass(DstReg));
+
+ MachineInstr *FirstFCMLAUpdated =
+ BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(IndexedFCMLAOpc),
+ DstReg)
+ .addReg(AccReg)
+ .addReg(RHSReg)
+ .addReg(DupSrcReg)
+ .addImm(DupSrcLane)
+ .addImm(FirstRotation)
+ .setMIFlags(MI.getFlags());
+ LLVM_DEBUG(dbgs() << MI << " replace by:\n: " << *FirstFCMLAUpdated << "\n");
+ MI.eraseFromParent();
+
+ DstReg = ParentFCMLA->getOperand(0).getReg();
+ Register ParentAccReg = ParentFCMLA->getOperand(1).getReg();
+ MRI->constrainRegClass(ParentAccReg, MRI->getRegClass(DstReg));
+ MachineInstr *SecondFCMLAUpdated =
+ BuildMI(*ParentFCMLA->getParent(), *ParentFCMLA,
+ ParentFCMLA->getDebugLoc(), TII->get(IndexedFCMLAOpc), DstReg)
+ .addReg(ParentAccReg)
+ .addReg(RHSReg)
+ .addReg(DupSrcReg)
+ .addImm(DupSrcLane)
+ .addImm(SecondRotation)
+ .setMIFlags(ParentFCMLA->getFlags());
+
+ LLVM_DEBUG(dbgs() << *ParentFCMLA
+ << " replace by:\n: " << *SecondFCMLAUpdated << "\n");
+
+ ParentFCMLA->eraseFromParent();
+ unsigned UsesCount = std::distance(MRI->use_nodbg_operands(LHSReg).begin(),
+ MRI->use_nodbg_operands(LHSReg).end());
+ if (UsesCount == 0) {
+ LLVM_DEBUG(dbgs() << "Removing " << *Dup << "\n");
+ Dup->eraseFromParent();
+ }
+ return true;
+}
+
bool AArch64MIPeepholeOptImpl::run(MachineFunction &MF) {
TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
TRI = static_cast<const AArch64RegisterInfo *>(
@@ -1049,6 +1175,11 @@ bool AArch64MIPeepholeOptImpl::run(MachineFunction &MF) {
case AArch64::COPY:
Changed |= visitCopy(MI);
break;
+ case AArch64::FCMLAv8f16:
+ case AArch64::FCMLAv4f16:
+ case AArch64::FCMLAv4f32:
+ Changed |= visitFCMLA(MI);
+ break;
}
}
}
diff --git a/llvm/test/CodeGen/AArch64/neon-vcmla.ll b/llvm/test/CodeGen/AArch64/neon-vcmla.ll
index 1d1ee1d1d56e1..851e4a8ee1684 100644
--- a/llvm/test/CodeGen/AArch64/neon-vcmla.ll
+++ b/llvm/test/CodeGen/AArch64/neon-vcmla.ll
@@ -614,6 +614,171 @@ entry:
ret <2 x double> %res
}
+define <4 x half> @test_rots_0_90_16x4_lhs_lane_1(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+; CHECK-LE-LABEL: test_rots_0_90_16x4_lhs_lane_1:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-NEXT: fcmla v0.4h, v2.4h, v1.h[1], #0
+; CHECK-LE-NEXT: fcmla v0.4h, v2.4h, v1.h[1], #90
+; CHECK-LE-NEXT: ret
+entry:
+ %b.cast = bitcast <4 x half> %b to <2 x i32>
+ %b.dup = shufflevector <2 x i32> %b.cast , <2 x i32> %b.cast, <2 x i32> <i32 1, i32 1>
+ %b.res = bitcast <2 x i32> %b.dup to <4 x half>
+ %res = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot0.v4f16(<4 x half> %a, <4 x half> %b.res, <4 x half> %c)
+ %res.two = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot90.v4f16(<4 x half> %res, <4 x half> %b.res, <4 x half> %c)
+ ret <4 x half> %res.two
+}
+
+define <4 x half> @test_rots_180_90_16x4_lhs_lane_0(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+; CHECK-LE-LABEL: test_rots_180_90_16x4_lhs_lane_0:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-NEXT: fcmla v0.4h, v2.4h, v1.h[0], #180
+; CHECK-LE-NEXT: fcmla v0.4h, v2.4h, v1.h[0], #90
+; CHECK-LE-NEXT: ret
+entry:
+ %b.cast = bitcast <4 x half> %b to <2 x i32>
+ %b.dup = shufflevector <2 x i32> %b.cast , <2 x i32> %b.cast, <2 x i32> <i32 0, i32 0>
+ %b.res = bitcast <2 x i32> %b.dup to <4 x half>
+ %res = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot180.v4f16(<4 x half> %a, <4 x half> %b.res, <4 x half> %c)
+ %res.two = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot90.v4f16(<4 x half> %res, <4 x half> %b.res, <4 x half> %c)
+ ret <4 x half> %res.two
+}
+
+
+define <8 x half> @test_rots_0_270_16x8_lhs_lane_3(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LE-LABEL: test_rots_0_270_16x8_lhs_lane_3:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: fcmla v0.8h, v2.8h, v1.h[3], #0
+; CHECK-LE-NEXT: fcmla v0.8h, v2.8h, v1.h[3], #270
+; CHECK-LE-NEXT: ret
+entry:
+ %b.cast = bitcast <8 x half> %b to <4 x i32>
+ %b.dup = shufflevector <4 x i32> %b.cast , <4 x i32> %b.cast, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+ %b.res = bitcast <4 x i32> %b.dup to <8 x half>
+ %res = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot0.v8f16(<8 x half> %a, <8 x half> %b.res, <8 x half> %c)
+ %res.two = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot270.v8f16(<8 x half> %res, <8 x half> %b.res, <8 x half> %c)
+ ret <8 x half> %res.two
+}
+
+define <8 x half> @test_rots_180_270_16x8_lhs_lane_2(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LE-LABEL: test_rots_180_270_16x8_lhs_lane_2:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: fcmla v0.8h, v2.8h, v1.h[2], #180
+; CHECK-LE-NEXT: fcmla v0.8h, v2.8h, v1.h[2], #270
+; CHECK-LE-NEXT: ret
+entry:
+ %b.cast = bitcast <8 x half> %b to <4 x i32>
+ %b.dup = shufflevector <4 x i32> %b.cast , <4 x i32> %b.cast, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+ %b.res = bitcast <4 x i32> %b.dup to <8 x half>
+ %res = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot180.v8f16(<8 x half> %a, <8 x half> %b.res, <8 x half> %c)
+ %res.two = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot270.v8f16(<8 x half> %res, <8 x half> %b.res, <8 x half> %c)
+ ret <8 x half> %res.two
+}
+
+define <4 x float> @test_rots_0_90_32x4_lhs_lane_0(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LE-LABEL: test_rots_0_90_32x4_lhs_lane_0:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: fcmla v0.4s, v2.4s, v1.s[0], #0
+; CHECK-LE-NEXT: fcmla v0.4s, v2.4s, v1.s[0], #90
+; CHECK-LE-NEXT: ret
+entry:
+ %b.cast = bitcast <4 x float> %b to <2 x i64>
+ %b.dup = shufflevector <2 x i64> %b.cast , <2 x i64> %b.cast, <2 x i32> <i32 0, i32 0>
+ %b.res = bitcast <2 x i64> %b.dup to <4 x float>
+ %res = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot0.v4f32(<4 x float> %a, <4 x float> %b.res, <4 x float> %c)
+ %res.two = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot90.v4f32(<4 x float> %res, <4 x float> %b.res, <4 x float> %c)
+ ret <4 x float> %res.two
+}
+
+define <4 x float> @test_rots_270_0_32x4_dup_lhs_im(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LE-LABEL: test_rots_270_0_32x4_dup_lhs_im:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: dup v1.4s, v1.s[1]
+; CHECK-LE-NEXT: fcmla v0.4s, v1.4s, v2.4s, #270
+; CHECK-LE-NEXT: fcmla v0.4s, v1.4s, v2.4s, #0
+; CHECK-LE-NOT: fcmla {{.*}}[1]
+; CHECK-LE-NEXT: ret
+entry:
+ %b.dup = shufflevector <4 x float> %b , <4 x float> %b, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+ %res = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot270.v4f32(<4 x float> %a, <4 x float> %b.dup, <4 x float> %c)
+ %res.two = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot0.v4f32(<4 x float> %res, <4 x float> %b.dup, <4 x float> %c)
+ ret <4 x float> %res.two
+}
+
+define <4 x half> @test_16x4_lane_1_lhs_dup_4users(<4 x half> %a, <4 x half> %b, <4 x half> %c, <4 x half> %d) {
+; CHECK-LE-LABEL: test_16x4_lane_1_lhs_dup_4users:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-NEXT: fcmla v0.4h, v2.4h, v1.h[1], #0
+; CHECK-LE-NEXT: fcmla v0.4h, v2.4h, v1.h[1], #90
+; CHECK-LE-NEXT: fcmla v0.4h, v3.4h, v1.h[1], #180
+; CHECK-LE-NEXT: fcmla v0.4h, v3.4h, v1.h[1], #270
+; CHECK-LE-NEXT: ret
+entry:
+ %b.cast = bitcast <4 x half> %b to <2 x i32>
+ %b.dup = shufflevector <2 x i32> %b.cast , <2 x i32> %b.cast, <2 x i32> <i32 1, i32 1>
+ %b.res = bitcast <2 x i32> %b.dup to <4 x half>
+ %res = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot0.v4f16(<4 x half> %a, <4 x half> %b.res, <4 x half> %c)
+ %res.two = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot90.v4f16(<4 x half> %res, <4 x half> %b.res, <4 x half> %c)
+ %res.three = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot180.v4f16(<4 x half> %res.two, <4 x half> %b.res, <4 x half> %d)
+ %res.four = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot270.v4f16(<4 x half> %res.three, <4 x half> %b.res, <4 x half> %d)
+ ret <4 x half> %res.four
+}
+
+
+define <4 x half> @test_16x4_lane_1_lhs_diff_operands(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+; CHECK-LE-LABEL: test_16x4_lane_1_lhs_diff_operands:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-NEXT: dup v3.2s, v1.s[1]
+; CHECK-LE-NEXT: fmov d1, d0
+; CHECK-LE-NEXT: fcmla v1.4h, v3.4h, v2.4h, #0
+; CHECK-LE-NEXT: fcmla v1.4h, v3.4h, v0.4h, #90
+; CHECK-LE-NOT: fcmla {{.*}}[1]
+entry:
+ %b.cast = bitcast <4 x half> %b to <2 x i32>
+ %b.dup = shufflevector <2 x i32> %b.cast , <2 x i32> %b.cast, <2 x i32> <i32 1, i32 1>
+ %b.res = bitcast <2 x i32> %b.dup to <4 x half>
+ %res = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot0.v4f16(<4 x half> %a, <4 x half> %b.res, <4 x half> %c)
+ %res.two = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot90.v4f16(<4 x half> %res, <4 x half> %b.res, <4 x half> %a)
+ ret <4 x half> %res.two
+}
+
+define <4 x half> @test_rots_0_0_16x4_lane_1_lhs(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+; CHECK-LE-LABEL: test_rots_0_0_16x4_lane_1_lhs:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-NEXT: dup v1.2s, v1.s[1]
+; CHECK-LE-NEXT: fcmla v0.4h, v1.4h, v2.4h, #0
+; CHECK-LE-NEXT: fcmla v0.4h, v1.4h, v2.4h, #0
+; CHECK-LE-NOT fcmla {{.*}}[1]
+entry:
+ %b.cast = bitcast <4 x half> %b to <2 x i32>
+ %b.dup = shufflevector <2 x i32> %b.cast , <2 x i32> %b.cast, <2 x i32> <i32 1, i32 1>
+ %b.res = bitcast <2 x i32> %b.dup to <4 x half>
+ %res = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot0.v4f16(<4 x half> %a, <4 x half> %b.res, <4 x half> %c)
+ %res.two = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot0.v4f16(<4 x half> %res, <4 x half> %b.res, <4 x half> %c)
+ ret <4 x half> %res.two
+}
+
+define <8 x half> @test_rots_90_270_16x8_lane_0_lhs(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LE-LABEL: test_rots_90_270_16x8_lane_0_lhs:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: dup v1.4s, v1.s[0]
+; CHECK-LE-NEXT: fcmla v0.8h, v1.8h, v2.8h, #90
+; CHECK-LE-NEXT: fcmla v0.8h, v1.8h, v2.8h, #270
+; CHECK-LE-NOT fcmla {{.*}}[0], #90
+entry:
+ %b.cast = bitcast <8 x half> %b to <4 x i32>
+ %b.dup = shufflevector <4 x i32> %b.cast , <4 x i32> %b.cast, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+ %b.res = bitcast <4 x i32> %b.dup to <8 x half>
+ %res = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot90.v8f16(<8 x half> %a, <8 x half> %b.res, <8 x half> %c)
+ %res.two = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot270.v8f16(<8 x half> %res, <8 x half> %b.res, <8 x half> %c)
+ ret <8 x half> %res.two
+}
+
define <4 x float> @reassoc_f32x4(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LE-SD-LABEL: reassoc_f32x4:
; CHECK-LE-SD: // %bb.0: // %entry
More information about the llvm-commits
mailing list