[llvm-branch-commits] [llvm] [AMDGPU] Prefer fma over fmac for f16 operands in the high half (PR #242072)

Dmitry Sidorov via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Thu Nov 12 15:49:57 PST 2026


https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/242072

>From c7efb12e40017c4b7c451341362a6f6cd1b1b8aa Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Thu, 12 Nov 2026 18:06:42 +0100
Subject: [PATCH 1/3] [AMDGPU] Prefer fma over fmac for f16 operands in the
 high half

Without real true16 instructions fmac has no op_sel, so a high half
operand needs a shift. v_fma_f16 reads it in place.
---
 llvm/lib/Target/AMDGPU/AMDGPUGISel.td         |   4 +
 llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp |  10 +
 llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h   |   1 +
 .../AMDGPU/AMDGPUInstructionSelector.cpp      |  11 ++
 .../Target/AMDGPU/AMDGPUInstructionSelector.h |   1 +
 llvm/lib/Target/AMDGPU/SIInstrInfo.td         |   2 +
 llvm/lib/Target/AMDGPU/SIInstructions.td      |  18 +-
 llvm/test/CodeGen/AMDGPU/fma.f16.ll           | 177 ++++++------------
 llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll         |   8 +-
 .../AMDGPU/fmul-2-combine-multi-use.ll        |   6 +-
 llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll    |  24 +--
 11 files changed, 108 insertions(+), 154 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 3c9c596b9c652..e7070294c3c25 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -39,6 +39,10 @@ def gi_vop3_no_mods :
     GIComplexOperandMatcher<s32, "selectVOP3NoMods">,
     GIComplexPatternEquiv<VOP3NoMods>;
 
+def gi_vop3_no_op_sel_mods :
+    GIComplexOperandMatcher<s32, "selectVOP3NoOpSelMods">,
+    GIComplexPatternEquiv<VOP3NoOpSelMods>;
+
 def gi_vop3omods :
     GIComplexOperandMatcher<s32, "selectVOP3OMods">,
     GIComplexPatternEquiv<VOP3OMods>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index c28c913ed9843..d3528321ff6a3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3675,6 +3675,16 @@ bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
   return true;
 }
 
+bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In,
+                                               SDValue &Src) const {
+  SDValue Unused;
+  if (!Subtarget->useRealTrue16Insts() && In.getValueSizeInBits() == 16 &&
+      isExtractHiElt(In, Unused))
+    return false;
+
+  return SelectVOP3NoMods(In, Src);
+}
+
 bool AMDGPUDAGToDAGISel::SelectVINTERPModsImpl(SDValue In, SDValue &Src,
                                                SDValue &SrcMods,
                                                bool OpSel) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
index ca587544b0ea6..f97f2e8645063 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
@@ -227,6 +227,7 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
                                        SDValue &SrcMods) const;
   bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
   bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
+  bool SelectVOP3NoOpSelMods(SDValue In, SDValue &Src) const;
   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
                        SDValue &Clamp, SDValue &Omod) const;
   bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index c0c90b3fcd1c6..c7af406dab056 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -4948,6 +4948,17 @@ AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
   }};
 }
 
+InstructionSelector::ComplexRendererFns
+AMDGPUInstructionSelector::selectVOP3NoOpSelMods(MachineOperand &Root) const {
+  Register Reg = Root.getReg();
+  Register Unused;
+  if (!Subtarget->useRealTrue16Insts() &&
+      MRI->getType(Reg).getSizeInBits() == 16 &&
+      isExtractHiElt(*MRI, Reg, Unused))
+    return {};
+  return selectVOP3NoMods(Root);
+}
+
 enum class SrcStatus {
   IS_SAME,
   IS_UPPER_HALF,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index b12ac778fd085..f7b9c29745570 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -191,6 +191,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
   selectVOP3BMods(MachineOperand &Root) const;
 
   ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
+  ComplexRendererFns selectVOP3NoOpSelMods(MachineOperand &Root) const;
 
   std::pair<Register, unsigned>
   selectVOP3PModsImpl(Register RootReg, const MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index c4dfed816f10a..1b1254d01b297 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -1754,6 +1754,8 @@ def VOP3ModsNonCanonicalizing : ComplexPattern<untyped, 2,
 
 def VOP3NoMods : ComplexPattern<untyped, 1, "SelectVOP3NoMods">;
 
+def VOP3NoOpSelMods : ComplexPattern<untyped, 1, "SelectVOP3NoOpSelMods">;
+
 def VOP3OMods : ComplexPattern<untyped, 3, "SelectVOP3OMods">;
 
 def VOP3PMods  : ComplexPattern<untyped, 2, "SelectVOP3PMods">;
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index c8c42b2809caa..4bd62c6c2107e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3945,25 +3945,25 @@ let SubtargetPredicate = isGFX10Plus in {
 // better to select fma instead of fmac.
 let True16Predicate = NotHasTrue16BitInsts in
 def : GCNPat <
-  (fma (f16 (VOP3NoMods f32:$src0)),
-       (f16 (VOP3NoMods f32:$src1)),
-       (f16 (VOP3NoMods f32:$src2))),
+  (fma (f16 (VOP3NoOpSelMods f32:$src0)),
+       (f16 (VOP3NoOpSelMods f32:$src1)),
+       (f16 (VOP3NoOpSelMods f32:$src2))),
   (V_FMAC_F16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,
                   SRCMODS.NONE, $src2)
 >;
 let True16Predicate = UseRealTrue16Insts in
 def : GCNPat <
-  (fma (f16 (VOP3NoMods f16:$src0)),
-       (f16 (VOP3NoMods f16:$src1)),
-       (f16 (VOP3NoMods f16:$src2))),
+  (fma (f16 (VOP3NoOpSelMods f16:$src0)),
+       (f16 (VOP3NoOpSelMods f16:$src1)),
+       (f16 (VOP3NoOpSelMods f16:$src2))),
   (V_FMAC_F16_t16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,
                   SRCMODS.NONE, $src2)
 >;
 let True16Predicate = UseFakeTrue16Insts in
 def : GCNPat <
-  (fma (f16 (VOP3NoMods f16:$src0)),
-       (f16 (VOP3NoMods f16:$src1)),
-       (f16 (VOP3NoMods f16:$src2))),
+  (fma (f16 (VOP3NoOpSelMods f16:$src0)),
+       (f16 (VOP3NoOpSelMods f16:$src1)),
+       (f16 (VOP3NoOpSelMods f16:$src2))),
   (V_FMAC_F16_fake16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,
                   SRCMODS.NONE, $src2)
 >;
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
index b517ad5f7eee1..27cec03324f64 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
@@ -796,8 +796,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) {
 ; GFX10-LABEL: test_fma_hi_src0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT:    v_fma_f16 v0, v0, v1, v2
+; GFX10-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0:
@@ -809,9 +808,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) {
 ; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src0:
 ; GFX11-SDAG-FAKE16:       ; %bb.0:
 ; GFX11-SDAG-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2
+; GFX11-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX11-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src0:
@@ -823,9 +820,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) {
 ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0:
 ; GFX11-GISEL-FAKE16:       ; %bb.0:
 ; GFX11-GISEL-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2
+; GFX11-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX11-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0:
@@ -845,9 +840,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) {
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX12-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src0:
@@ -867,9 +860,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) {
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2
+; GFX12-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX12-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %x1 = extractelement <2 x half> %x, i32 1
   %r = call half @llvm.fma.f16(half %x1, half %y, half %z)
@@ -886,10 +877,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) {
 ; GFX10-LABEL: test_fma_hi_srcs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
-; GFX10-NEXT:    v_fmac_f16_e32 v0, v3, v1
+; GFX10-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_srcs:
@@ -901,11 +889,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) {
 ; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_srcs:
 ; GFX11-SDAG-FAKE16:       ; %bb.0:
 ; GFX11-SDAG-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_fmac_f16_e32 v0, v3, v1
+; GFX11-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0]
 ; GFX11-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_srcs:
@@ -917,11 +901,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) {
 ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_srcs:
 ; GFX11-GISEL-FAKE16:       ; %bb.0:
 ; GFX11-GISEL-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
-; GFX11-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT:    v_fmac_f16_e32 v0, v3, v1
+; GFX11-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0]
 ; GFX11-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_srcs:
@@ -941,11 +921,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) {
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_fmac_f16_e32 v0, v3, v1
+; GFX12-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0]
 ; GFX12-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_srcs:
@@ -965,11 +941,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) {
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX12-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
-; GFX12-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT:    v_fmac_f16_e32 v0, v3, v1
+; GFX12-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0]
 ; GFX12-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %x1 = extractelement <2 x half> %x, i32 1
   %y1 = extractelement <2 x half> %y, i32 1
@@ -989,9 +961,8 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) {
 ; GFX10-LABEL: test_fma_hi_src0_two_uses:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX10-NEXT:    v_fma_f16 v0, v3, v1, v2
-; GFX10-NEXT:    v_fmac_f16_e32 v0, v3, v2
+; GFX10-NEXT:    v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
+; GFX10-NEXT:    v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0_two_uses:
@@ -1005,10 +976,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) {
 ; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src0_two_uses:
 ; GFX11-SDAG-FAKE16:       ; %bb.0:
 ; GFX11-SDAG-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v3, v1, v2
-; GFX11-SDAG-FAKE16-NEXT:    v_fmac_f16_e32 v0, v3, v2
+; GFX11-SDAG-FAKE16-NEXT:    v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0]
 ; GFX11-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src0_two_uses:
@@ -1022,10 +992,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) {
 ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0_two_uses:
 ; GFX11-GISEL-FAKE16:       ; %bb.0:
 ; GFX11-GISEL-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v3, v1, v2
-; GFX11-GISEL-FAKE16-NEXT:    v_fmac_f16_e32 v0, v3, v2
+; GFX11-GISEL-FAKE16-NEXT:    v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
+; GFX11-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0]
 ; GFX11-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0_two_uses:
@@ -1047,10 +1016,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) {
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v3, v1, v2
-; GFX12-SDAG-FAKE16-NEXT:    v_fmac_f16_e32 v0, v3, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0]
 ; GFX12-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src0_two_uses:
@@ -1072,10 +1040,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) {
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v3, v1, v2
-; GFX12-GISEL-FAKE16-NEXT:    v_fmac_f16_e32 v0, v3, v2
+; GFX12-GISEL-FAKE16-NEXT:    v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
+; GFX12-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0]
 ; GFX12-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %x1 = extractelement <2 x half> %x, i32 1
   %r0 = call half @llvm.fma.f16(half %x1, half %y, half %z)
@@ -1093,9 +1060,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) {
 ; GFX10-LABEL: test_fma_hi_src2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX10-NEXT:    v_fmac_f16_e32 v2, v0, v1
-; GFX10-NEXT:    v_mov_b32_e32 v0, v2
+; GFX10-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src2:
@@ -1107,10 +1072,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) {
 ; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src2:
 ; GFX11-SDAG-FAKE16:       ; %bb.0:
 ; GFX11-SDAG-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_fmac_f16_e32 v2, v0, v1
-; GFX11-SDAG-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0]
 ; GFX11-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src2:
@@ -1122,10 +1084,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) {
 ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src2:
 ; GFX11-GISEL-FAKE16:       ; %bb.0:
 ; GFX11-GISEL-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT:    v_fmac_f16_e32 v2, v0, v1
-; GFX11-GISEL-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0]
 ; GFX11-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src2:
@@ -1145,10 +1104,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) {
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_fmac_f16_e32 v2, v0, v1
-; GFX12-SDAG-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0]
 ; GFX12-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src2:
@@ -1168,10 +1124,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) {
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT:    v_fmac_f16_e32 v2, v0, v1
-; GFX12-GISEL-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX12-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0]
 ; GFX12-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %z1 = extractelement <2 x half> %z, i32 1
   %r = call half @llvm.fma.f16(half %x, half %y, half %z1)
@@ -1200,7 +1153,7 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-SDAG-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
-; GFX10-SDAG-NEXT:    v_fma_f16 v0, v5, v1, v2
+; GFX10-SDAG-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX10-SDAG-NEXT:    flat_store_short v[3:4], v5
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -1208,9 +1161,8 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
 ; GFX10-GISEL-LABEL: test_fma_hi_src0_stored:
 ; GFX10-GISEL:       ; %bb.0:
 ; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
+; GFX10-GISEL-NEXT:    v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX10-GISEL-NEXT:    flat_store_short_d16_hi v[3:4], v0
-; GFX10-GISEL-NEXT:    v_fma_f16 v1, v5, v1, v2
 ; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
@@ -1227,8 +1179,7 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
 ; GFX11-SDAG-FAKE16:       ; %bb.0:
 ; GFX11-SDAG-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v5, v1, v2
+; GFX11-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX11-SDAG-FAKE16-NEXT:    flat_store_b16 v[3:4], v5
 ; GFX11-SDAG-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -1247,10 +1198,8 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
 ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0_stored:
 ; GFX11-GISEL-FAKE16:       ; %bb.0:
 ; GFX11-GISEL-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-GISEL-FAKE16-NEXT:    v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX11-GISEL-FAKE16-NEXT:    flat_store_d16_hi_b16 v[3:4], v0
-; GFX11-GISEL-FAKE16-NEXT:    v_fma_f16 v1, v5, v1, v2
-; GFX11-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-GISEL-FAKE16-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX11-GISEL-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -1275,8 +1224,7 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v5, v1, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX12-SDAG-FAKE16-NEXT:    flat_store_b16 v[3:4], v5
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -1303,10 +1251,8 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
+; GFX12-GISEL-FAKE16-NEXT:    v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX12-GISEL-FAKE16-NEXT:    flat_store_d16_hi_b16 v[3:4], v0
-; GFX12-GISEL-FAKE16-NEXT:    v_fma_f16 v1, v5, v1, v2
-; GFX12-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-FAKE16-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -1334,8 +1280,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) {
 ; GFX10-LABEL: test_fma_hi_src0_literal_src1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT:    v_fmamk_f16 v0, v0, 0x6400, v1
+; GFX10-NEXT:    v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0_literal_src1:
@@ -1347,9 +1292,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) {
 ; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src0_literal_src1:
 ; GFX11-SDAG-FAKE16:       ; %bb.0:
 ; GFX11-SDAG-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_fmamk_f16 v0, v0, 0x6400, v1
+; GFX11-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0]
 ; GFX11-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src0_literal_src1:
@@ -1361,9 +1304,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) {
 ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0_literal_src1:
 ; GFX11-GISEL-FAKE16:       ; %bb.0:
 ; GFX11-GISEL-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT:    v_fmamk_f16 v0, v0, 0x6400, v1
+; GFX11-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0]
 ; GFX11-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0_literal_src1:
@@ -1383,9 +1324,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) {
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_fmamk_f16 v0, v0, 0x6400, v1
+; GFX12-SDAG-FAKE16-NEXT:    v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0]
 ; GFX12-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src0_literal_src1:
@@ -1405,9 +1344,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) {
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT:    v_fmamk_f16 v0, v0, 0x6400, v1
+; GFX12-GISEL-FAKE16-NEXT:    v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0]
 ; GFX12-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %x1 = extractelement <2 x half> %x, i32 1
   %r = call half @llvm.fma.f16(half %x1, half 1024.0, half %z)
@@ -1460,14 +1397,14 @@ define half @test_fma_hi_src0_other_block(<2 x half> %x, half %y, half %z, i1 %c
 ; GFX10-GISEL:       ; %bb.0:
 ; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-GISEL-NEXT:    v_and_b32_e32 v3, 1, v3
-; GFX10-GISEL-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX10-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX10-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX10-GISEL-NEXT:    s_and_saveexec_b32 s4, vcc_lo
 ; GFX10-GISEL-NEXT:  ; %bb.1: ; %fma
-; GFX10-GISEL-NEXT:    v_fmac_f16_e32 v2, v0, v1
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, v2
+; GFX10-GISEL-NEXT:    v_fma_f16 v3, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX10-GISEL-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
 ; GFX10-GISEL-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0_other_block:
@@ -1525,16 +1462,16 @@ define half @test_fma_hi_src0_other_block(<2 x half> %x, half %y, half %z, i1 %c
 ; GFX11-GISEL-FAKE16:       ; %bb.0:
 ; GFX11-GISEL-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-FAKE16-NEXT:    v_and_b32_e32 v3, 1, v3
-; GFX11-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT:    s_mov_b32 s0, exec_lo
-; GFX11-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT:    v_cmpx_ne_u32_e32 0, v3
+; GFX11-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX11-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-GISEL-FAKE16-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-GISEL-FAKE16-NEXT:  ; %bb.1: ; %fma
-; GFX11-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT:    v_fmac_f16_e32 v2, v0, v1
-; GFX11-GISEL-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-GISEL-FAKE16-NEXT:    v_fma_f16 v3, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX11-GISEL-FAKE16-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
 ; GFX11-GISEL-FAKE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX11-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0_other_block:
@@ -1611,17 +1548,17 @@ define half @test_fma_hi_src0_other_block(<2 x half> %x, half %y, half %z, i1 %c
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-FAKE16-NEXT:    v_and_b32_e32 v3, 1, v3
-; GFX12-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-GISEL-FAKE16-NEXT:    s_mov_b32 s0, exec_lo
-; GFX12-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-GISEL-FAKE16-NEXT:    v_cmpx_ne_u32_e32 0, v3
+; GFX12-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX12-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX12-GISEL-FAKE16-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX12-GISEL-FAKE16-NEXT:  ; %bb.1: ; %fma
-; GFX12-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT:    v_fmac_f16_e32 v2, v0, v1
-; GFX12-GISEL-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX12-GISEL-FAKE16-NEXT:    v_fma_f16 v3, v0, v1, v2 op_sel:[1,0,0,0]
 ; GFX12-GISEL-FAKE16-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
 ; GFX12-GISEL-FAKE16-NEXT:    s_wait_alu depctr_sa_sdst(0)
 ; GFX12-GISEL-FAKE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX12-GISEL-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %x1 = extractelement <2 x half> %x, i32 1
   br i1 %c, label %fma, label %exit
diff --git a/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll b/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
index ed62bf07ecd25..96ac9557f78dc 100644
--- a/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
@@ -1,9 +1,11 @@
 ; RUN: llc -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GCN,GFX1010 %s
-; RUN: llc -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=GCN,GFX1010 %s
+; RUN: llc -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=GCN,GFX1100 %s
 
 ; GCN-LABEL: {{^}}addMul2D:
-; GFX1010: v_fmac_f16
-; GFX1010: v_fmac_f16
+; GFX1010: v_fma_f16 {{.*}} op_sel:[1,0,1,0]
+; GFX1010: v_fma_f16 {{.*}} op_sel:[1,0,1,0]
+; GFX1100: v_fmac_f16
+; GFX1100: v_fmac_f16
 define hidden <4 x half> @addMul2D(ptr nocapture readonly %arg, ptr addrspace(4) nocapture readonly %arg1, <2 x i32> %arg2, i32 %arg3) local_unnamed_addr #0 {
 bb:
   %tmp = extractelement <2 x i32> %arg2, i64 1
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
index b0270a50790aa..f0f4f5627a5a2 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
@@ -601,9 +601,8 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
 ; GFX10-DENORM-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-DENORM-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DENORM-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-NEXT:    s_lshr_b32 s3, s2, 16
 ; GFX10-DENORM-NEXT:    v_add_f16_e64 v1, s2, s2
-; GFX10-DENORM-NEXT:    v_fma_f16 v2, s2, 2.0, s3
+; GFX10-DENORM-NEXT:    v_fma_f16 v2, s2, 2.0, s2 op_sel:[0,0,1,0]
 ; GFX10-DENORM-NEXT:    global_store_short v0, v1, s[0:1]
 ; GFX10-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-DENORM-NEXT:    global_store_short v0, v2, s[0:1] offset:2
@@ -649,9 +648,8 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
 ; GFX11-DENORM-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
 ; GFX11-DENORM-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11-DENORM-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-DENORM-FAKE16-NEXT:    s_lshr_b32 s3, s2, 16
 ; GFX11-DENORM-FAKE16-NEXT:    v_add_f16_e64 v1, s2, s2
-; GFX11-DENORM-FAKE16-NEXT:    v_fma_f16 v2, s2, 2.0, s3
+; GFX11-DENORM-FAKE16-NEXT:    v_fma_f16 v2, s2, 2.0, s2 op_sel:[0,0,1,0]
 ; GFX11-DENORM-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1] dlc
 ; GFX11-DENORM-FAKE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-DENORM-FAKE16-NEXT:    global_store_b16 v0, v2, s[0:1] offset:2 dlc
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll
index 9c9d6e01e5c06..65e8c1e6d0fc9 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll
@@ -148,18 +148,12 @@ define <4 x half> @v_constained_fma_v4f16_fpexcept_strict(<4 x half> %x, <4 x ha
 ; GFX10-LABEL: v_constained_fma_v4f16_fpexcept_strict:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v0
+; GFX10-NEXT:    v_fma_f16 v6, v1, v3, v5 op_sel:[1,1,1,0]
+; GFX10-NEXT:    v_fma_f16 v7, v0, v2, v4 op_sel:[1,1,1,0]
 ; GFX10-NEXT:    v_fmac_f16_e32 v4, v0, v2
-; GFX10-NEXT:    v_fmac_f16_e32 v6, v8, v7
 ; GFX10-NEXT:    v_fmac_f16_e32 v5, v1, v3
-; GFX10-NEXT:    v_fmac_f16_e32 v9, v11, v10
+; GFX10-NEXT:    v_perm_b32 v0, v7, v4, 0x5040100
 ; GFX10-NEXT:    v_perm_b32 v1, v6, v5, 0x5040100
-; GFX10-NEXT:    v_perm_b32 v0, v9, v4, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_constained_fma_v4f16_fpexcept_strict:
@@ -175,18 +169,12 @@ define <4 x half> @v_constained_fma_v4f16_fpexcept_strict(<4 x half> %x, <4 x ha
 ; GFX11-FAKE16-LABEL: v_constained_fma_v4f16_fpexcept_strict:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v4
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v0
+; GFX11-FAKE16-NEXT:    v_fma_f16 v6, v1, v3, v5 op_sel:[1,1,1,0]
+; GFX11-FAKE16-NEXT:    v_fma_f16 v7, v0, v2, v4 op_sel:[1,1,1,0]
 ; GFX11-FAKE16-NEXT:    v_fmac_f16_e32 v4, v0, v2
-; GFX11-FAKE16-NEXT:    v_fmac_f16_e32 v6, v8, v7
 ; GFX11-FAKE16-NEXT:    v_fmac_f16_e32 v5, v1, v3
-; GFX11-FAKE16-NEXT:    v_fmac_f16_e32 v9, v11, v10
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v7, v4, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v6, v5, 0x5040100
-; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v9, v4, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %val = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> %x, <4 x half> %y, <4 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
   ret <4 x half> %val

>From 90c0408a8ccbae43b2b6485086f6d429ff757303 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Thu, 12 Nov 2026 23:05:37 +0100
Subject: [PATCH 2/3] format

---
 llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index d3528321ff6a3..e9508757c6c96 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3675,8 +3675,7 @@ bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
   return true;
 }
 
-bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In,
-                                               SDValue &Src) const {
+bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In, SDValue &Src) const {
   SDValue Unused;
   if (!Subtarget->useRealTrue16Insts() && In.getValueSizeInBits() == 16 &&
       isExtractHiElt(In, Unused))

>From 660c8010b35465f2f4c721f1e65a55c56f64be3a Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 13 Nov 2026 00:40:44 +0100
Subject: [PATCH 3/3] apply review

---
 llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp        | 3 +--
 llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp | 5 +----
 llvm/lib/Target/AMDGPU/SIInstructions.td             | 6 +++---
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index e9508757c6c96..ef0d11b190cde 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3677,8 +3677,7 @@ bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
 
 bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In, SDValue &Src) const {
   SDValue Unused;
-  if (!Subtarget->useRealTrue16Insts() && In.getValueSizeInBits() == 16 &&
-      isExtractHiElt(In, Unused))
+  if (isExtractHiElt(In, Unused))
     return false;
 
   return SelectVOP3NoMods(In, Src);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index c7af406dab056..4f444b2a8735b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -4950,11 +4950,8 @@ AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
 
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectVOP3NoOpSelMods(MachineOperand &Root) const {
-  Register Reg = Root.getReg();
   Register Unused;
-  if (!Subtarget->useRealTrue16Insts() &&
-      MRI->getType(Reg).getSizeInBits() == 16 &&
-      isExtractHiElt(*MRI, Reg, Unused))
+  if (isExtractHiElt(*MRI, Root.getReg(), Unused))
     return {};
   return selectVOP3NoMods(Root);
 }
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 4bd62c6c2107e..916baa0a978ab 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3953,9 +3953,9 @@ def : GCNPat <
 >;
 let True16Predicate = UseRealTrue16Insts in
 def : GCNPat <
-  (fma (f16 (VOP3NoOpSelMods f16:$src0)),
-       (f16 (VOP3NoOpSelMods f16:$src1)),
-       (f16 (VOP3NoOpSelMods f16:$src2))),
+  (fma (f16 (VOP3NoMods f16:$src0)),
+       (f16 (VOP3NoMods f16:$src1)),
+       (f16 (VOP3NoMods f16:$src2))),
   (V_FMAC_F16_t16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,
                   SRCMODS.NONE, $src2)
 >;



More information about the llvm-branch-commits mailing list