[llvm] [AArch64] Use SVE/NEON FMLAL top/bottom instructions (PR #186798)
Sander de Smalen via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 16 06:33:09 PDT 2026
https://github.com/sdesmalen-arm created https://github.com/llvm/llvm-project/pull/186798
None
>From 413629a92ee21f7f0e0deb9360f5acd9c4aaf235 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Mon, 16 Mar 2026 10:41:11 +0000
Subject: [PATCH 1/2] [AArch64] Use NEON FMLAL(2) for partial FADD reductions.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 2 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 7 +++
.../AArch64/f16f32dot-fixed-length-fdot.ll | 57 +++++++++++++++----
3 files changed, 55 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c1a6654f68b2b..2c1205960ed24 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1509,7 +1509,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
}
}
- if (Subtarget->hasF16F32DOT()) {
+ if (Subtarget->hasF16F32DOT() || Subtarget->hasFP16FML()) {
setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, MVT::v2f32,
MVT::v4f16, Legal);
setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, MVT::v4f32,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 68b85f07194a2..198cbd66c3224 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1824,6 +1824,13 @@ defm FMLALlane : SIMDThreeSameVectorFMLIndex<0, 0b0000, "fmlal", int_aarch64_ne
defm FMLSLlane : SIMDThreeSameVectorFMLIndex<0, 0b0100, "fmlsl", int_aarch64_neon_fmlsl>;
defm FMLAL2lane : SIMDThreeSameVectorFMLIndex<1, 0b1000, "fmlal2", int_aarch64_neon_fmlal2>;
defm FMLSL2lane : SIMDThreeSameVectorFMLIndex<1, 0b1100, "fmlsl2", int_aarch64_neon_fmlsl2>;
+
+def : Pat<(v2f32 (partial_reduce_fmla v2f32:$acc, v4f16:$LHS, v4f16:$RHS)),
+ (FMLAL2v4f16 (FMLALv4f16 v2f32:$acc, V64:$LHS, V64:$RHS),
+ V64:$LHS, V64:$RHS)>;
+def : Pat<(v4f32 (partial_reduce_fmla v4f32:$acc, v8f16:$LHS, v8f16:$RHS)),
+ (FMLAL2v8f16 (FMLALv8f16 v4f32:$acc, V128:$LHS, V128:$RHS),
+ V128:$LHS, V128:$RHS)>;
}
// Armv8.2-A Crypto extensions
diff --git a/llvm/test/CodeGen/AArch64/f16f32dot-fixed-length-fdot.ll b/llvm/test/CodeGen/AArch64/f16f32dot-fixed-length-fdot.ll
index 2645cfdfc0c89..5b1fe56244fbe 100644
--- a/llvm/test/CodeGen/AArch64/f16f32dot-fixed-length-fdot.ll
+++ b/llvm/test/CodeGen/AArch64/f16f32dot-fixed-length-fdot.ll
@@ -1,13 +1,34 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mattr=+f16f32dot < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK-EXPAND
+; RUN: llc -mattr=+fp16fml < %s | FileCheck %s --check-prefixes=CHECK-FMLAL
+; RUN: llc -mattr=+f16f32dot < %s | FileCheck %s --check-prefixes=CHECK-FDOT
+; RUN: llc -mattr=+f16f32dot,+fp16fml < %s | FileCheck %s --check-prefixes=CHECK-FDOT
target triple = "aarch64-linux-gnu"
define <4 x float> @fixed_fdot_wide(<4 x float> %acc, <8 x half> %a, <8 x half> %b) {
-; CHECK-LABEL: fixed_fdot_wide:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fdot v0.4s, v1.8h, v2.8h
-; CHECK-NEXT: ret
+; CHECK-EXPAND-LABEL: fixed_fdot_wide:
+; CHECK-EXPAND: // %bb.0: // %entry
+; CHECK-EXPAND-NEXT: fcvtl v3.4s, v1.4h
+; CHECK-EXPAND-NEXT: fcvtl v4.4s, v2.4h
+; CHECK-EXPAND-NEXT: fcvtl2 v1.4s, v1.8h
+; CHECK-EXPAND-NEXT: fcvtl2 v2.4s, v2.8h
+; CHECK-EXPAND-NEXT: fmul v3.4s, v3.4s, v4.4s
+; CHECK-EXPAND-NEXT: fmul v1.4s, v1.4s, v2.4s
+; CHECK-EXPAND-NEXT: fadd v0.4s, v0.4s, v3.4s
+; CHECK-EXPAND-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-EXPAND-NEXT: ret
+;
+; CHECK-FMLAL-LABEL: fixed_fdot_wide:
+; CHECK-FMLAL: // %bb.0: // %entry
+; CHECK-FMLAL-NEXT: fmlal v0.4s, v1.4h, v2.4h
+; CHECK-FMLAL-NEXT: fmlal2 v0.4s, v1.4h, v2.4h
+; CHECK-FMLAL-NEXT: ret
+;
+; CHECK-FDOT-LABEL: fixed_fdot_wide:
+; CHECK-FDOT: // %bb.0: // %entry
+; CHECK-FDOT-NEXT: fdot v0.4s, v1.8h, v2.8h
+; CHECK-FDOT-NEXT: ret
entry:
%a.wide = fpext <8 x half> %a to <8 x float>
%b.wide = fpext <8 x half> %b to <8 x float>
@@ -17,14 +38,30 @@ entry:
}
define <2 x float> @fixed_fdot(<2 x float> %acc, <4 x half> %a, <4 x half> %b) {
-; CHECK-LABEL: fixed_fdot:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fdot v0.2s, v1.4h, v2.4h
-; CHECK-NEXT: ret
+; CHECK-EXPAND-LABEL: fixed_fdot:
+; CHECK-EXPAND: // %bb.0: // %entry
+; CHECK-EXPAND-NEXT: fcvtl v1.4s, v1.4h
+; CHECK-EXPAND-NEXT: fcvtl v2.4s, v2.4h
+; CHECK-EXPAND-NEXT: fmul v1.4s, v1.4s, v2.4s
+; CHECK-EXPAND-NEXT: fadd v0.2s, v0.2s, v1.2s
+; CHECK-EXPAND-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-EXPAND-NEXT: fadd v0.2s, v1.2s, v0.2s
+; CHECK-EXPAND-NEXT: ret
+;
+; CHECK-FMLAL-LABEL: fixed_fdot:
+; CHECK-FMLAL: // %bb.0: // %entry
+; CHECK-FMLAL-NEXT: fmlal v0.2s, v1.2h, v2.2h
+; CHECK-FMLAL-NEXT: fmlal2 v0.2s, v1.2h, v2.2h
+; CHECK-FMLAL-NEXT: ret
+;
+; CHECK-FDOT-LABEL: fixed_fdot:
+; CHECK-FDOT: // %bb.0: // %entry
+; CHECK-FDOT-NEXT: fdot v0.2s, v1.4h, v2.4h
+; CHECK-FDOT-NEXT: ret
entry:
%a.wide = fpext <4 x half> %a to <4 x float>
%b.wide = fpext <4 x half> %b to <4 x float>
%mult = fmul <4 x float> %a.wide, %b.wide
%partial.reduce = call <2 x float> @llvm.vector.partial.reduce.fadd(<2 x float> %acc, <4 x float> %mult)
ret <2 x float> %partial.reduce
-}
\ No newline at end of file
+}
>From 2f11802b261f621fba1d4f93dea897559e7c9500 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Mon, 16 Mar 2026 13:17:59 +0000
Subject: [PATCH 2/2] [AArch64] Use SVE FMLALB/T for partial FADD reductions.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 2 +-
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 4 ++
llvm/test/CodeGen/AArch64/sve2p1-fdot.ll | 25 +++--------
.../AArch64/sve2p1-fixed-length-fdot.ll | 42 ++++++++-----------
4 files changed, 27 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2c1205960ed24..7e2448f7511e1 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2039,7 +2039,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
}
// Handle floating-point partial reduction
- if (Subtarget->hasSVE2p1() || Subtarget->hasSME2()) {
+ if (Subtarget->hasSVE2() || Subtarget->hasSME()) {
setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, MVT::nxv4f32,
MVT::nxv8f16, Legal);
// We can use SVE2p1 fdot to emulate the fixed-length variant.
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 926593022b537..3386cabd9fdf6 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4162,6 +4162,10 @@ let Predicates = [HasSVE2_or_SME] in {
defm BSL2N_ZZZZ : sve2_int_bitwise_ternary_op<0b101, "bsl2n", AArch64bsl2n>;
defm NBSL_ZZZZ : sve2_int_bitwise_ternary_op<0b111, "nbsl", AArch64nbsl>;
+ def : Pat<(nxv4f32 (partial_reduce_fmla nxv4f32:$acc, nxv8f16:$LHS, nxv8f16:$RHS)),
+ (FMLALT_ZZZ_SHH (FMLALB_ZZZ_SHH nxv4f32:$acc, ZPR:$LHS, ZPR:$RHS),
+ ZPR:$LHS, ZPR:$RHS)>;
+
// Pseudos for commutable operations that expand to NBSL or BSL2N. These can
// avoid a COPY if one of the source registers can be clobbered, or otherwise
// use a MOVPRFX.
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-fdot.ll b/llvm/test/CodeGen/AArch64/sve2p1-fdot.ll
index 9dbe096ebdb57..d3954dc6bf8d6 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-fdot.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-fdot.ll
@@ -9,19 +9,8 @@ target triple = "aarch64-linux-gnu"
define <vscale x 4 x float> @fdot_wide_nxv4f32(<vscale x 4 x float> %acc, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
; SVE2-LABEL: fdot_wide_nxv4f32:
; SVE2: // %bb.0: // %entry
-; SVE2-NEXT: uunpklo z3.s, z1.h
-; SVE2-NEXT: uunpklo z4.s, z2.h
-; SVE2-NEXT: ptrue p0.s
-; SVE2-NEXT: uunpkhi z1.s, z1.h
-; SVE2-NEXT: uunpkhi z2.s, z2.h
-; SVE2-NEXT: fcvt z3.s, p0/m, z3.h
-; SVE2-NEXT: fcvt z4.s, p0/m, z4.h
-; SVE2-NEXT: fcvt z1.s, p0/m, z1.h
-; SVE2-NEXT: fcvt z2.s, p0/m, z2.h
-; SVE2-NEXT: fmul z3.s, z3.s, z4.s
-; SVE2-NEXT: fmul z1.s, z1.s, z2.s
-; SVE2-NEXT: fadd z0.s, z0.s, z3.s
-; SVE2-NEXT: fadd z0.s, z0.s, z1.s
+; SVE2-NEXT: fmlalb z0.s, z1.h, z2.h
+; SVE2-NEXT: fmlalt z0.s, z1.h, z2.h
; SVE2-NEXT: ret
;
; SVE2P1-LABEL: fdot_wide_nxv4f32:
@@ -39,13 +28,9 @@ entry:
define <vscale x 4 x float> @fdot_splat_nxv4f32(<vscale x 4 x float> %acc, <vscale x 8 x half> %a) {
; SVE2-LABEL: fdot_splat_nxv4f32:
; SVE2: // %bb.0: // %entry
-; SVE2-NEXT: uunpklo z2.s, z1.h
-; SVE2-NEXT: ptrue p0.s
-; SVE2-NEXT: uunpkhi z1.s, z1.h
-; SVE2-NEXT: fcvt z2.s, p0/m, z2.h
-; SVE2-NEXT: fcvt z1.s, p0/m, z1.h
-; SVE2-NEXT: fadd z0.s, z0.s, z2.s
-; SVE2-NEXT: fadd z0.s, z0.s, z1.s
+; SVE2-NEXT: fmov z2.h, #1.00000000
+; SVE2-NEXT: fmlalb z0.s, z1.h, z2.h
+; SVE2-NEXT: fmlalt z0.s, z1.h, z2.h
; SVE2-NEXT: ret
;
; SVE2P1-LABEL: fdot_splat_nxv4f32:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-fixed-length-fdot.ll b/llvm/test/CodeGen/AArch64/sve2p1-fixed-length-fdot.ll
index 4463b072f69cf..76b39ba429a0d 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-fixed-length-fdot.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-fixed-length-fdot.ll
@@ -7,17 +7,11 @@ target triple = "aarch64-linux-gnu"
define void @fdot_v4f32(ptr %accptr, ptr %aptr, ptr %bptr) {
; SVE2-LABEL: fdot_v4f32:
; SVE2: // %bb.0: // %entry
-; SVE2-NEXT: ldr q0, [x1]
-; SVE2-NEXT: ldr q1, [x2]
-; SVE2-NEXT: fcvtl v2.4s, v0.4h
-; SVE2-NEXT: fcvtl v3.4s, v1.4h
-; SVE2-NEXT: fcvtl2 v0.4s, v0.8h
-; SVE2-NEXT: fcvtl2 v1.4s, v1.8h
-; SVE2-NEXT: fmul v2.4s, v2.4s, v3.4s
-; SVE2-NEXT: ldr q3, [x0]
-; SVE2-NEXT: fmul v0.4s, v0.4s, v1.4s
-; SVE2-NEXT: fadd v1.4s, v3.4s, v2.4s
-; SVE2-NEXT: fadd v0.4s, v1.4s, v0.4s
+; SVE2-NEXT: ldr q0, [x0]
+; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: ldr q2, [x2]
+; SVE2-NEXT: fmlalb z0.s, z1.h, z2.h
+; SVE2-NEXT: fmlalt z0.s, z1.h, z2.h
; SVE2-NEXT: str q0, [x0]
; SVE2-NEXT: ret
;
@@ -216,14 +210,12 @@ entry:
define <4 x float> @fixed_fdot_wide(<4 x float> %acc, <8 x half> %a, <8 x half> %b) {
; SVE2-LABEL: fixed_fdot_wide:
; SVE2: // %bb.0: // %entry
-; SVE2-NEXT: fcvtl v3.4s, v1.4h
-; SVE2-NEXT: fcvtl v4.4s, v2.4h
-; SVE2-NEXT: fcvtl2 v1.4s, v1.8h
-; SVE2-NEXT: fcvtl2 v2.4s, v2.8h
-; SVE2-NEXT: fmul v3.4s, v3.4s, v4.4s
-; SVE2-NEXT: fmul v1.4s, v1.4s, v2.4s
-; SVE2-NEXT: fadd v0.4s, v0.4s, v3.4s
-; SVE2-NEXT: fadd v0.4s, v0.4s, v1.4s
+; SVE2-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT: // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT: fmlalb z0.s, z1.h, z2.h
+; SVE2-NEXT: fmlalt z0.s, z1.h, z2.h
+; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE2-NEXT: ret
;
; SVE2P1-LABEL: fixed_fdot_wide:
@@ -245,12 +237,12 @@ entry:
define <2 x float> @fixed_fdot(<2 x float> %acc, <4 x half> %a, <4 x half> %b) {
; SVE2-LABEL: fixed_fdot:
; SVE2: // %bb.0: // %entry
-; SVE2-NEXT: fcvtl v1.4s, v1.4h
-; SVE2-NEXT: fcvtl v2.4s, v2.4h
-; SVE2-NEXT: fmul v1.4s, v1.4s, v2.4s
-; SVE2-NEXT: fadd v0.2s, v0.2s, v1.2s
-; SVE2-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; SVE2-NEXT: fadd v0.2s, v1.2s, v0.2s
+; SVE2-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT: // kill: def $d2 killed $d2 def $z2
+; SVE2-NEXT: // kill: def $d1 killed $d1 def $z1
+; SVE2-NEXT: fmlalb z0.s, z1.h, z2.h
+; SVE2-NEXT: fmlalt z0.s, z1.h, z2.h
+; SVE2-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE2-NEXT: ret
;
; SVE2P1-LABEL: fixed_fdot:
More information about the llvm-commits
mailing list