[llvm] [Hexagon] Add qf16 fused multiply-accumulate patterns (PR #215656)

Tue Aug 11 13:47:44 PDT 2026

https://github.com/chamikasudusinghe created https://github.com/llvm/llvm-project/pull/215656

On HVX with the qfloat feature an f16 multiply produces a qf16 result (`V6_vmpy_qf16_hf`), and `V6_vadd_qf16_mix` accumulates a qf16 value with an hf value while keeping the result in qf16. Lowering an `fmul` and an `fadd` separately converts the product back to hf in between, so a multiply-accumulate takes four instructions instead of three.

This makes `ISD::FMA` legal for `v64f16` under `UseHVXQFloat` and adds a pattern that keeps the product in qf16 across the accumulate, converting to hf once at the end. DAGCombine contracts `fmul`+`fadd` into the `fma` where the fast-math flags allow it, so the loop-carried accumulator of a vectorized f16 matmul inner loop drops the extra conversion per MAC.

Test: `llvm/test/CodeGen/Hexagon/autohvx/qf16-fused-mac.ll` covers an explicit `llvm.fma.v64f16` and a fast-math loop-carried MAC.

>From 458eef3e49dbbe556f2315514a2170873ac6fe29 Mon Sep 17 00:00:00 2001
From: Chamika Sudusinghe <csudusin at qti.qualcomm.com>
Date: Tue, 11 Aug 2026 15:40:17 -0500
Subject: [PATCH] [Hexagon] Add qf16 fused multiply-accumulate patterns

On HVX with the qfloat feature an f16 multiply produces a qf16 result
(V6_vmpy_qf16_hf), and V6_vadd_qf16_mix accumulates a qf16 value with an hf
value while keeping the result in qf16. Lowering an fmul and an fadd
separately converts the product back to hf in between, so a multiply-add
takes four instructions instead of three.

Make ISD::FMA legal for v64f16 under UseHVXQFloat and add a pattern that
keeps the product in qf16 across the accumulate, converting to hf once at the
end. DAGCombine contracts fmul+fadd into the fma where the fast-math flags
allow it.
---
 .../Target/Hexagon/HexagonISelLoweringHVX.cpp |  4 ++
 llvm/lib/Target/Hexagon/HexagonPatternsHVX.td |  8 ++++
 .../CodeGen/Hexagon/autohvx/qf16-fused-mac.ll | 46 +++++++++++++++++++
 3 files changed, 58 insertions(+)
 create mode 100644 llvm/test/CodeGen/Hexagon/autohvx/qf16-fused-mac.ll

diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index 670fe270f36a1..df853d1c083a5 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -249,6 +249,10 @@ HexagonTargetLowering::initializeHVXLowering() {
     if (Subtarget.useHVXQFloatOps()) {
       setOperationAction(ISD::FP_EXTEND, MVT::v64f32, Custom);
       setOperationAction(ISD::FP_ROUND,  MVT::v64f16, Legal);
+      // qf16 has a fused multiply-accumulate form (see HexagonPatternsHVX.td).
+      // Declaring it legal lets DAGCombine contract fmul+fadd into fma where
+      // the fast-math flags permit it, instead of emitting both separately.
+      setOperationAction(ISD::FMA, MVT::v64f16, Legal);
     } else if (Subtarget.useHVXIEEEFPOps()) {
       setOperationAction(ISD::FP_EXTEND, MVT::v64f32, Legal);
       setOperationAction(ISD::FP_ROUND,  MVT::v64f16, Legal);
diff --git a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
index fa13bd013bc7c..48015af46f045 100644
--- a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
+++ b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
@@ -565,6 +565,14 @@ let Predicates = [UseHVXV68, UseHVXQFloat] in {
   def: OpR_RR_pat_conv_hf<V6_vsub_hf,        pf2<fsub>,  VecF16, HVF16>;
   def: OpR_RR_pat_conv_hf<V6_vadd_hf,        pf2<fadd>,  VecF16, HVF16>;
   def: OpR_RR_pat_conv_hf<V6_vmpy_qf16_hf,   pf2<fmul>,  VecF16, HVF16>;
+
+  // Fused multiply-add. Keeping the vmpy product in qf16 and accumulating with
+  // vadd_qf16_mix avoids the qf16->hf conversion that lowering the multiply and
+  // the add separately would emit, so a MAC is 3 instructions instead of 4.
+  def: Pat<(VecF16 (fma HVF16:$Vu, HVF16:$Vv, HVF16:$Vacc)),
+           (V6_vconv_hf_qf16 (V6_vadd_qf16_mix
+             (V6_vmpy_qf16_hf HVF16:$Vu, HVF16:$Vv), HVF16:$Vacc))>;
+
   def: OpR_RR_pat_conv<V6_vsub_sf,        pf2<fsub>,  VecF32, HVF32>;
   def: OpR_RR_pat_conv<V6_vadd_sf,        pf2<fadd>,  VecF32, HVF32>;
   def: OpR_RR_pat_conv<V6_vmpy_qf32_sf,   pf2<fmul>,  VecF32, HVF32>;
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/qf16-fused-mac.ll b/llvm/test/CodeGen/Hexagon/autohvx/qf16-fused-mac.ll
new file mode 100644
index 0000000000000..7afa87cd70d97
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/autohvx/qf16-fused-mac.ll
@@ -0,0 +1,46 @@
+; RUN: llc -mtriple=hexagon < %s | FileCheck %s
+
+; Fused multiply-accumulate on qf16: the vmpy product stays in qf16 and is
+; accumulated with vadd_qf16_mix, so a MAC needs one conversion back to hf
+; instead of two.
+
+; The shape that matters: a loop-carried accumulator, which is what a vectorized
+; f16 matmul inner loop looks like. Without the fma pattern the product is
+; converted back to hf before the add (vadd(hf,hf)), costing an extra
+; instruction per MAC.
+define <64 x half> @mac_loop_carried(<64 x half> %init, ptr %pa, ptr %pb, i32 %n) #0 {
+; CHECK-LABEL: mac_loop_carried:
+; CHECK:         [[M:v[0-9]+]].qf16 = vmpy({{v[0-9]+}}.hf,{{v[0-9]+}}.hf)
+; CHECK:         [[A:v[0-9]+]].qf16 = vadd([[M]].qf16,{{v[0-9]+}}.hf)
+; CHECK-NOT:     .hf = [[M]].qf16
+entry:
+  br label %loop
+
+loop:
+  %acc = phi <64 x half> [ %init, %entry ], [ %new, %loop ]
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %a = load <64 x half>, ptr %pa
+  %b = load <64 x half>, ptr %pb
+  %mul = fmul fast <64 x half> %a, %b
+  %new = fadd fast <64 x half> %acc, %mul
+  %i.next = add i32 %i, 1
+  %cmp = icmp slt i32 %i.next, %n
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret <64 x half> %new
+}
+
+; An explicit fma is matched directly, with no fast-math flags needed.
+define <64 x half> @fma_v64f16(<64 x half> %a0, <64 x half> %a1, <64 x half> %a2) #0 {
+; CHECK-LABEL: fma_v64f16:
+; CHECK:         [[M:v[0-9]+]].qf16 = vmpy(v0.hf,v1.hf)
+; CHECK:         [[A:v[0-9]+]].qf16 = vadd([[M]].qf16,v2.hf)
+; CHECK:         v0.hf = [[A]].qf16
+  %v0 = call <64 x half> @llvm.fma.v64f16(<64 x half> %a0, <64 x half> %a1, <64 x half> %a2)
+  ret <64 x half> %v0
+}
+
+declare <64 x half> @llvm.fma.v64f16(<64 x half>, <64 x half>, <64 x half>)
+
+attributes #0 = { nounwind "target-cpu"="hexagonv68" "target-features"="+hvxv68,+hvx-length128b,+hvx-qfloat" }

