[llvm] [Hexagon] Add qf16 fused multiply-accumulate patterns (PR #215656)

[via llvm-commits]

llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-backend-hexagon

Author: Chamika Sudusinghe (chamikasudusinghe)

<details>
<summary>Changes</summary>

On HVX with the qfloat feature an f16 multiply produces a qf16 result (`V6_vmpy_qf16_hf`), and `V6_vadd_qf16_mix` accumulates a qf16 value with an hf value while keeping the result in qf16. Lowering an `fmul` and an `fadd` separately converts the product back to hf in between, so a multiply-accumulate takes four instructions instead of three.

This makes `ISD::FMA` legal for `v64f16` under `UseHVXQFloat` and adds a pattern that keeps the product in qf16 across the accumulate, converting to hf once at the end. DAGCombine contracts `fmul`+`fadd` into the `fma` where the fast-math flags allow it, so the loop-carried accumulator of a vectorized f16 matmul inner loop drops the extra conversion per MAC.

Test: `llvm/test/CodeGen/Hexagon/autohvx/qf16-fused-mac.ll` covers an explicit `llvm.fma.v64f16` and a fast-math loop-carried MAC.

---
Full diff: https://github.com/llvm/llvm-project/pull/215656.diff


3 Files Affected:

- (modified) llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp (+4) 
- (modified) llvm/lib/Target/Hexagon/HexagonPatternsHVX.td (+8) 
- (added) llvm/test/CodeGen/Hexagon/autohvx/qf16-fused-mac.ll (+46) 


``````````diff
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index 670fe270f36a1..df853d1c083a5 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -249,6 +249,10 @@ HexagonTargetLowering::initializeHVXLowering() {
     if (Subtarget.useHVXQFloatOps()) {
       setOperationAction(ISD::FP_EXTEND, MVT::v64f32, Custom);
       setOperationAction(ISD::FP_ROUND,  MVT::v64f16, Legal);
+      // qf16 has a fused multiply-accumulate form (see HexagonPatternsHVX.td).
+      // Declaring it legal lets DAGCombine contract fmul+fadd into fma where
+      // the fast-math flags permit it, instead of emitting both separately.
+      setOperationAction(ISD::FMA, MVT::v64f16, Legal);
     } else if (Subtarget.useHVXIEEEFPOps()) {
       setOperationAction(ISD::FP_EXTEND, MVT::v64f32, Legal);
       setOperationAction(ISD::FP_ROUND,  MVT::v64f16, Legal);
diff --git a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
index fa13bd013bc7c..48015af46f045 100644
--- a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
+++ b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
@@ -565,6 +565,14 @@ let Predicates = [UseHVXV68, UseHVXQFloat] in {
   def: OpR_RR_pat_conv_hf<V6_vsub_hf,        pf2<fsub>,  VecF16, HVF16>;
   def: OpR_RR_pat_conv_hf<V6_vadd_hf,        pf2<fadd>,  VecF16, HVF16>;
   def: OpR_RR_pat_conv_hf<V6_vmpy_qf16_hf,   pf2<fmul>,  VecF16, HVF16>;
+
+  // Fused multiply-add. Keeping the vmpy product in qf16 and accumulating with
+  // vadd_qf16_mix avoids the qf16->hf conversion that lowering the multiply and
+  // the add separately would emit, so a MAC is 3 instructions instead of 4.
+  def: Pat<(VecF16 (fma HVF16:$Vu, HVF16:$Vv, HVF16:$Vacc)),
+           (V6_vconv_hf_qf16 (V6_vadd_qf16_mix
+             (V6_vmpy_qf16_hf HVF16:$Vu, HVF16:$Vv), HVF16:$Vacc))>;
+
   def: OpR_RR_pat_conv<V6_vsub_sf,        pf2<fsub>,  VecF32, HVF32>;
   def: OpR_RR_pat_conv<V6_vadd_sf,        pf2<fadd>,  VecF32, HVF32>;
   def: OpR_RR_pat_conv<V6_vmpy_qf32_sf,   pf2<fmul>,  VecF32, HVF32>;
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/qf16-fused-mac.ll b/llvm/test/CodeGen/Hexagon/autohvx/qf16-fused-mac.ll
new file mode 100644
index 0000000000000..7afa87cd70d97
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/autohvx/qf16-fused-mac.ll
@@ -0,0 +1,46 @@
+; RUN: llc -mtriple=hexagon < %s | FileCheck %s
+
+; Fused multiply-accumulate on qf16: the vmpy product stays in qf16 and is
+; accumulated with vadd_qf16_mix, so a MAC needs one conversion back to hf
+; instead of two.
+
+; The shape that matters: a loop-carried accumulator, which is what a vectorized
+; f16 matmul inner loop looks like. Without the fma pattern the product is
+; converted back to hf before the add (vadd(hf,hf)), costing an extra
+; instruction per MAC.
+define <64 x half> @mac_loop_carried(<64 x half> %init, ptr %pa, ptr %pb, i32 %n) #0 {
+; CHECK-LABEL: mac_loop_carried:
+; CHECK:         [[M:v[0-9]+]].qf16 = vmpy({{v[0-9]+}}.hf,{{v[0-9]+}}.hf)
+; CHECK:         [[A:v[0-9]+]].qf16 = vadd([[M]].qf16,{{v[0-9]+}}.hf)
+; CHECK-NOT:     .hf = [[M]].qf16
+entry:
+  br label %loop
+
+loop:
+  %acc = phi <64 x half> [ %init, %entry ], [ %new, %loop ]
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %a = load <64 x half>, ptr %pa
+  %b = load <64 x half>, ptr %pb
+  %mul = fmul fast <64 x half> %a, %b
+  %new = fadd fast <64 x half> %acc, %mul
+  %i.next = add i32 %i, 1
+  %cmp = icmp slt i32 %i.next, %n
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret <64 x half> %new
+}
+
+; An explicit fma is matched directly, with no fast-math flags needed.
+define <64 x half> @fma_v64f16(<64 x half> %a0, <64 x half> %a1, <64 x half> %a2) #0 {
+; CHECK-LABEL: fma_v64f16:
+; CHECK:         [[M:v[0-9]+]].qf16 = vmpy(v0.hf,v1.hf)
+; CHECK:         [[A:v[0-9]+]].qf16 = vadd([[M]].qf16,v2.hf)
+; CHECK:         v0.hf = [[A]].qf16
+  %v0 = call <64 x half> @llvm.fma.v64f16(<64 x half> %a0, <64 x half> %a1, <64 x half> %a2)
+  ret <64 x half> %v0
+}
+
+declare <64 x half> @llvm.fma.v64f16(<64 x half>, <64 x half>, <64 x half>)
+
+attributes #0 = { nounwind "target-cpu"="hexagonv68" "target-features"="+hvxv68,+hvx-length128b,+hvx-qfloat" }

``````````

</details>


https://github.com/llvm/llvm-project/pull/215656