[llvm] [AMDGPU] Add tests for scalar hi-half f16 fma and mad on GFX9 (PR #219432)
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 28 03:20:17 PDT 2026
https://github.com/jayfoad created https://github.com/llvm/llvm-project/pull/219432
v_fma_f16 and v_mad_legacy_f16 both support op_sel on GFX9, so a 16-bit ternary
whose sources come from the high halves of packed registers could be a single
instruction. SelectVOP3OpSel() does not set op_sel yet, so the high halves are
extracted and re-inserted with explicit shifts instead. Add tests for both SDAG
and GlobalISel so that the improvement is visible if this is ever fixed.
Co-Authored-By: Claude Opus 5 (1M context) <noreply at anthropic.com>
>From 4221b609ddcb4ff1b622ca44f6852fddb6a6c318 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Fri, 28 Aug 2026 11:14:46 +0100
Subject: [PATCH] [AMDGPU] Add tests for scalar hi-half f16 fma and mad on GFX9
v_fma_f16 and v_mad_legacy_f16 both support op_sel on GFX9, so a 16-bit ternary
whose sources come from the high halves of packed registers could be a single
instruction. SelectVOP3OpSel() does not set op_sel yet, so the high halves are
extracted and re-inserted with explicit shifts instead. Add tests for both SDAG
and GlobalISel so that the improvement is visible if this is ever fixed.
Co-Authored-By: Claude Opus 5 (1M context) <noreply at anthropic.com>
---
llvm/test/CodeGen/AMDGPU/fma-mad-f16-hi16.ll | 176 +++++++++++++++++++
1 file changed, 176 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/fma-mad-f16-hi16.ll
diff --git a/llvm/test/CodeGen/AMDGPU/fma-mad-f16-hi16.ll b/llvm/test/CodeGen/AMDGPU/fma-mad-f16-hi16.ll
new file mode 100644
index 0000000000000..c3a6d58f0ca3d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fma-mad-f16-hi16.ll
@@ -0,0 +1,176 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9-GISEL %s
+
+; Scalar 16-bit VOP3 ternaries reading and writing the high half of a packed
+; register. On GFX9 v_fma_f16 and v_mad_legacy_f16 both support op_sel, so each
+; of these could be a single instruction, but SelectVOP3OpSel() does not set
+; op_sel yet (see the FIXME in AMDGPUISelDAGToDAG.cpp), so the high halves are
+; extracted and re-inserted with explicit shifts instead.
+
+; Only src0 comes from a high half.
+define half @fma_f16_hi_src0(<2 x half> %a, half %b, half %c) {
+; GFX9-SDAG-LABEL: fma_f16_hi_src0:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fma_f16_hi_src0:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %a1 = extractelement <2 x half> %a, i32 1
+ %r = call half @llvm.fma.f16(half %a1, half %b, half %c)
+ ret half %r
+}
+
+; All three sources come from high halves.
+define half @fma_f16_hi_srcs(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
+; GFX9-SDAG-LABEL: fma_f16_hi_srcs:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fma_f16_hi_srcs:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %a1 = extractelement <2 x half> %a, i32 1
+ %b1 = extractelement <2 x half> %b, i32 1
+ %c1 = extractelement <2 x half> %c, i32 1
+ %r = call half @llvm.fma.f16(half %a1, half %b1, half %c1)
+ ret half %r
+}
+
+; All three sources come from high halves, and the result goes back into a high
+; half, so op_sel could cover the destination too.
+define <2 x half> @fma_f16_hi_srcs_hi_dst(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
+; GFX9-SDAG-LABEL: fma_f16_hi_srcs_hi_dst:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, v1, v3
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v2, s4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fma_f16_hi_srcs_hi_dst:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, v3
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffff
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v2, v1, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %a1 = extractelement <2 x half> %a, i32 1
+ %b1 = extractelement <2 x half> %b, i32 1
+ %c1 = extractelement <2 x half> %c, i32 1
+ %r = call half @llvm.fma.f16(half %a1, half %b1, half %c1)
+ %v = insertelement <2 x half> %c, half %r, i32 1
+ ret <2 x half> %v
+}
+
+; Plain mad shrinks to the VOP2 form, which has no op_sel at all.
+define half @mad_f16_hi_srcs(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
+; GFX9-SDAG-LABEL: mad_f16_hi_srcs:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-SDAG-NEXT: v_mac_f16_e32 v0, v3, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: mad_f16_hi_srcs:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-GISEL-NEXT: v_mac_f16_e32 v0, v3, v1
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %a1 = extractelement <2 x half> %a, i32 1
+ %b1 = extractelement <2 x half> %b, i32 1
+ %c1 = extractelement <2 x half> %c, i32 1
+ %r = call half @llvm.amdgcn.fmad.ftz.f16(half %a1, half %b1, half %c1)
+ ret half %r
+}
+
+; A source modifier keeps this in the VOP3 form, which does have op_sel.
+define half @mad_f16_hi_srcs_fneg(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
+; GFX9-SDAG-LABEL: mad_f16_hi_srcs_fneg:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-SDAG-NEXT: v_mad_legacy_f16 v0, v0, -v1, v2
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: mad_f16_hi_srcs_fneg:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT: v_mad_legacy_f16 v0, v0, -v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %a1 = extractelement <2 x half> %a, i32 1
+ %b1 = extractelement <2 x half> %b, i32 1
+ %b1.neg = fneg half %b1
+ %c1 = extractelement <2 x half> %c, i32 1
+ %r = call half @llvm.amdgcn.fmad.ftz.f16(half %a1, half %b1.neg, half %c1)
+ ret half %r
+}
+
+; VOP3 mad with the result going back into a high half.
+define <2 x half> @mad_f16_hi_srcs_fneg_hi_dst(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
+; GFX9-SDAG-LABEL: mad_f16_hi_srcs_fneg_hi_dst:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX9-SDAG-NEXT: v_mad_legacy_f16 v0, v0, -v1, v3
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x5040100
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v2, s4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: mad_f16_hi_srcs_fneg_hi_dst:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX9-GISEL-NEXT: v_mad_legacy_f16 v0, v0, -v1, v3
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffff
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v2, v1, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %a1 = extractelement <2 x half> %a, i32 1
+ %b1 = extractelement <2 x half> %b, i32 1
+ %b1.neg = fneg half %b1
+ %c1 = extractelement <2 x half> %c, i32 1
+ %r = call half @llvm.amdgcn.fmad.ftz.f16(half %a1, half %b1.neg, half %c1)
+ %v = insertelement <2 x half> %c, half %r, i32 1
+ ret <2 x half> %v
+}
More information about the llvm-commits
mailing list