[llvm-branch-commits] [llvm] [AMDGPU] Prefer fma over fmac for f16 operands in the high half (PR #242072)
Dmitry Sidorov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Nov 13 05:05:37 PST 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/242072
>From dd98b04406aafd1e064ae0a2fc8ca570f738d9b3 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Thu, 12 Nov 2026 18:06:42 +0100
Subject: [PATCH 1/3] [AMDGPU] Prefer fma over fmac for f16 operands in the
high half
Without real true16 instructions fmac has no op_sel, so a high half
operand needs a shift. v_fma_f16 reads it in place.
---
llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 4 +
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 10 +
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h | 1 +
.../AMDGPU/AMDGPUInstructionSelector.cpp | 11 ++
.../Target/AMDGPU/AMDGPUInstructionSelector.h | 1 +
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 2 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 18 +-
llvm/test/CodeGen/AMDGPU/fma.f16.ll | 177 ++++++------------
llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll | 8 +-
.../AMDGPU/fmul-2-combine-multi-use.ll | 6 +-
llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll | 24 +--
11 files changed, 108 insertions(+), 154 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 3c9c596b9c652..e7070294c3c25 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -39,6 +39,10 @@ def gi_vop3_no_mods :
GIComplexOperandMatcher<s32, "selectVOP3NoMods">,
GIComplexPatternEquiv<VOP3NoMods>;
+def gi_vop3_no_op_sel_mods :
+ GIComplexOperandMatcher<s32, "selectVOP3NoOpSelMods">,
+ GIComplexPatternEquiv<VOP3NoOpSelMods>;
+
def gi_vop3omods :
GIComplexOperandMatcher<s32, "selectVOP3OMods">,
GIComplexPatternEquiv<VOP3OMods>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index afd8be7337742..eb8e342f2c33b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3680,6 +3680,16 @@ bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
return true;
}
+bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In,
+ SDValue &Src) const {
+ SDValue Unused;
+ if (!Subtarget->useRealTrue16Insts() && In.getValueSizeInBits() == 16 &&
+ isExtractHiElt(In, Unused))
+ return false;
+
+ return SelectVOP3NoMods(In, Src);
+}
+
bool AMDGPUDAGToDAGISel::SelectVINTERPModsImpl(SDValue In, SDValue &Src,
SDValue &SrcMods,
bool OpSel) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
index ca587544b0ea6..f97f2e8645063 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
@@ -227,6 +227,7 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
SDValue &SrcMods) const;
bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
+ bool SelectVOP3NoOpSelMods(SDValue In, SDValue &Src) const;
bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
SDValue &Clamp, SDValue &Omod) const;
bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index c0c90b3fcd1c6..c7af406dab056 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -4948,6 +4948,17 @@ AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
}};
}
+InstructionSelector::ComplexRendererFns
+AMDGPUInstructionSelector::selectVOP3NoOpSelMods(MachineOperand &Root) const {
+ Register Reg = Root.getReg();
+ Register Unused;
+ if (!Subtarget->useRealTrue16Insts() &&
+ MRI->getType(Reg).getSizeInBits() == 16 &&
+ isExtractHiElt(*MRI, Reg, Unused))
+ return {};
+ return selectVOP3NoMods(Root);
+}
+
enum class SrcStatus {
IS_SAME,
IS_UPPER_HALF,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index b12ac778fd085..f7b9c29745570 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -191,6 +191,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
selectVOP3BMods(MachineOperand &Root) const;
ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
+ ComplexRendererFns selectVOP3NoOpSelMods(MachineOperand &Root) const;
std::pair<Register, unsigned>
selectVOP3PModsImpl(Register RootReg, const MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index c4dfed816f10a..1b1254d01b297 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -1754,6 +1754,8 @@ def VOP3ModsNonCanonicalizing : ComplexPattern<untyped, 2,
def VOP3NoMods : ComplexPattern<untyped, 1, "SelectVOP3NoMods">;
+def VOP3NoOpSelMods : ComplexPattern<untyped, 1, "SelectVOP3NoOpSelMods">;
+
def VOP3OMods : ComplexPattern<untyped, 3, "SelectVOP3OMods">;
def VOP3PMods : ComplexPattern<untyped, 2, "SelectVOP3PMods">;
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index c8c42b2809caa..4bd62c6c2107e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3945,25 +3945,25 @@ let SubtargetPredicate = isGFX10Plus in {
// better to select fma instead of fmac.
let True16Predicate = NotHasTrue16BitInsts in
def : GCNPat <
- (fma (f16 (VOP3NoMods f32:$src0)),
- (f16 (VOP3NoMods f32:$src1)),
- (f16 (VOP3NoMods f32:$src2))),
+ (fma (f16 (VOP3NoOpSelMods f32:$src0)),
+ (f16 (VOP3NoOpSelMods f32:$src1)),
+ (f16 (VOP3NoOpSelMods f32:$src2))),
(V_FMAC_F16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,
SRCMODS.NONE, $src2)
>;
let True16Predicate = UseRealTrue16Insts in
def : GCNPat <
- (fma (f16 (VOP3NoMods f16:$src0)),
- (f16 (VOP3NoMods f16:$src1)),
- (f16 (VOP3NoMods f16:$src2))),
+ (fma (f16 (VOP3NoOpSelMods f16:$src0)),
+ (f16 (VOP3NoOpSelMods f16:$src1)),
+ (f16 (VOP3NoOpSelMods f16:$src2))),
(V_FMAC_F16_t16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,
SRCMODS.NONE, $src2)
>;
let True16Predicate = UseFakeTrue16Insts in
def : GCNPat <
- (fma (f16 (VOP3NoMods f16:$src0)),
- (f16 (VOP3NoMods f16:$src1)),
- (f16 (VOP3NoMods f16:$src2))),
+ (fma (f16 (VOP3NoOpSelMods f16:$src0)),
+ (f16 (VOP3NoOpSelMods f16:$src1)),
+ (f16 (VOP3NoOpSelMods f16:$src2))),
(V_FMAC_F16_fake16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,
SRCMODS.NONE, $src2)
>;
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
index b517ad5f7eee1..27cec03324f64 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
@@ -796,8 +796,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) {
; GFX10-LABEL: test_fma_hi_src0:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX10-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0:
@@ -809,9 +808,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) {
; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src0:
; GFX11-SDAG-FAKE16: ; %bb.0:
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src0:
@@ -823,9 +820,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) {
; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0:
@@ -845,9 +840,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src0:
@@ -867,9 +860,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%x1 = extractelement <2 x half> %x, i32 1
%r = call half @llvm.fma.f16(half %x1, half %y, half %z)
@@ -886,10 +877,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) {
; GFX10-LABEL: test_fma_hi_srcs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_fmac_f16_e32 v0, v3, v1
+; GFX10-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0]
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_srcs:
@@ -901,11 +889,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) {
; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_srcs:
; GFX11-SDAG-FAKE16: ; %bb.0:
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v1
+; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0]
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_srcs:
@@ -917,11 +901,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) {
; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_srcs:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v1
+; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0]
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_srcs:
@@ -941,11 +921,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v1
+; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0]
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_srcs:
@@ -965,11 +941,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v1
+; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0]
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%x1 = extractelement <2 x half> %x, i32 1
%y1 = extractelement <2 x half> %y, i32 1
@@ -989,9 +961,8 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) {
; GFX10-LABEL: test_fma_hi_src0_two_uses:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX10-NEXT: v_fma_f16 v0, v3, v1, v2
-; GFX10-NEXT: v_fmac_f16_e32 v0, v3, v2
+; GFX10-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
+; GFX10-NEXT: v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0]
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0_two_uses:
@@ -1005,10 +976,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) {
; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src0_two_uses:
; GFX11-SDAG-FAKE16: ; %bb.0:
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v3, v1, v2
-; GFX11-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v2
+; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0]
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src0_two_uses:
@@ -1022,10 +992,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) {
; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0_two_uses:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v3, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v2
+; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0]
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0_two_uses:
@@ -1047,10 +1016,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v3, v1, v2
-; GFX12-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v2
+; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0]
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src0_two_uses:
@@ -1072,10 +1040,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v3, v1, v2
-; GFX12-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v2
+; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0]
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%x1 = extractelement <2 x half> %x, i32 1
%r0 = call half @llvm.fma.f16(half %x1, half %y, half %z)
@@ -1093,9 +1060,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) {
; GFX10-LABEL: test_fma_hi_src2:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX10-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX10-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0]
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src2:
@@ -1107,10 +1072,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) {
; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src2:
; GFX11-SDAG-FAKE16: ; %bb.0:
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0]
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src2:
@@ -1122,10 +1084,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) {
; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src2:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0]
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src2:
@@ -1145,10 +1104,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0]
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src2:
@@ -1168,10 +1124,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0]
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%z1 = extractelement <2 x half> %z, i32 1
%r = call half @llvm.fma.f16(half %x, half %y, half %z1)
@@ -1200,7 +1153,7 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX10-SDAG-NEXT: v_fma_f16 v0, v5, v1, v2
+; GFX10-SDAG-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
; GFX10-SDAG-NEXT: flat_store_short v[3:4], v5
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -1208,9 +1161,8 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
; GFX10-GISEL-LABEL: test_fma_hi_src0_stored:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX10-GISEL-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
; GFX10-GISEL-NEXT: flat_store_short_d16_hi v[3:4], v0
-; GFX10-GISEL-NEXT: v_fma_f16 v1, v5, v1, v2
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v1
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1227,8 +1179,7 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
; GFX11-SDAG-FAKE16: ; %bb.0:
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v5, v1, v2
+; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
; GFX11-SDAG-FAKE16-NEXT: flat_store_b16 v[3:4], v5
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1247,10 +1198,8 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0_stored:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
; GFX11-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v[3:4], v0
-; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v1, v5, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1275,8 +1224,7 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v5, v1, v2
+; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0]
; GFX12-SDAG-FAKE16-NEXT: flat_store_b16 v[3:4], v5
; GFX12-SDAG-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1303,10 +1251,8 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0]
; GFX12-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v[3:4], v0
-; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v1, v5, v1, v2
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v1
; GFX12-GISEL-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1334,8 +1280,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) {
; GFX10-LABEL: test_fma_hi_src0_literal_src1:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_fmamk_f16 v0, v0, 0x6400, v1
+; GFX10-NEXT: v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0]
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0_literal_src1:
@@ -1347,9 +1292,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) {
; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src0_literal_src1:
; GFX11-SDAG-FAKE16: ; %bb.0:
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_fmamk_f16 v0, v0, 0x6400, v1
+; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0]
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src0_literal_src1:
@@ -1361,9 +1304,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) {
; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0_literal_src1:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_fmamk_f16 v0, v0, 0x6400, v1
+; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0]
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0_literal_src1:
@@ -1383,9 +1324,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_fmamk_f16 v0, v0, 0x6400, v1
+; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0]
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src0_literal_src1:
@@ -1405,9 +1344,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_fmamk_f16 v0, v0, 0x6400, v1
+; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0]
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%x1 = extractelement <2 x half> %x, i32 1
%r = call half @llvm.fma.f16(half %x1, half 1024.0, half %z)
@@ -1460,14 +1397,14 @@ define half @test_fma_hi_src0_other_block(<2 x half> %x, half %y, half %z, i1 %c
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 1, v3
-; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX10-GISEL-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-GISEL-NEXT: ; %bb.1: ; %fma
-; GFX10-GISEL-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-GISEL-NEXT: v_fma_f16 v3, v0, v1, v2 op_sel:[1,0,0,0]
; GFX10-GISEL-NEXT: ; %bb.2: ; %UnifiedReturnBlock
; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0_other_block:
@@ -1525,16 +1462,16 @@ define half @test_fma_hi_src0_other_block(<2 x half> %x, half %y, half %z, i1 %c
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v3, 1, v3
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_cmpx_ne_u32_e32 0, v3
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: ; %bb.1: ; %fma
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v3, v0, v1, v2 op_sel:[1,0,0,0]
; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %UnifiedReturnBlock
; GFX11-GISEL-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0_other_block:
@@ -1611,17 +1548,17 @@ define half @test_fma_hi_src0_other_block(<2 x half> %x, half %y, half %z, i1 %c
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v3, 1, v3
-; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-GISEL-FAKE16-NEXT: v_cmpx_ne_u32_e32 0, v3
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX12-GISEL-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX12-GISEL-FAKE16-NEXT: ; %bb.1: ; %fma
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v3, v0, v1, v2 op_sel:[1,0,0,0]
; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %UnifiedReturnBlock
; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v3
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%x1 = extractelement <2 x half> %x, i32 1
br i1 %c, label %fma, label %exit
diff --git a/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll b/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
index ed62bf07ecd25..96ac9557f78dc 100644
--- a/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
@@ -1,9 +1,11 @@
; RUN: llc -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GCN,GFX1010 %s
-; RUN: llc -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=GCN,GFX1010 %s
+; RUN: llc -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=GCN,GFX1100 %s
; GCN-LABEL: {{^}}addMul2D:
-; GFX1010: v_fmac_f16
-; GFX1010: v_fmac_f16
+; GFX1010: v_fma_f16 {{.*}} op_sel:[1,0,1,0]
+; GFX1010: v_fma_f16 {{.*}} op_sel:[1,0,1,0]
+; GFX1100: v_fmac_f16
+; GFX1100: v_fmac_f16
define hidden <4 x half> @addMul2D(ptr nocapture readonly %arg, ptr addrspace(4) nocapture readonly %arg1, <2 x i32> %arg2, i32 %arg3) local_unnamed_addr #0 {
bb:
%tmp = extractelement <2 x i32> %arg2, i64 1
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
index b0270a50790aa..f0f4f5627a5a2 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
@@ -601,9 +601,8 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
; GFX10-DENORM-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-DENORM-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DENORM-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-NEXT: s_lshr_b32 s3, s2, 16
; GFX10-DENORM-NEXT: v_add_f16_e64 v1, s2, s2
-; GFX10-DENORM-NEXT: v_fma_f16 v2, s2, 2.0, s3
+; GFX10-DENORM-NEXT: v_fma_f16 v2, s2, 2.0, s2 op_sel:[0,0,1,0]
; GFX10-DENORM-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DENORM-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-DENORM-NEXT: global_store_short v0, v2, s[0:1] offset:2
@@ -649,9 +648,8 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
; GFX11-DENORM-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-DENORM-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-DENORM-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-DENORM-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
; GFX11-DENORM-FAKE16-NEXT: v_add_f16_e64 v1, s2, s2
-; GFX11-DENORM-FAKE16-NEXT: v_fma_f16 v2, s2, 2.0, s3
+; GFX11-DENORM-FAKE16-NEXT: v_fma_f16 v2, s2, 2.0, s2 op_sel:[0,0,1,0]
; GFX11-DENORM-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1] dlc
; GFX11-DENORM-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-DENORM-FAKE16-NEXT: global_store_b16 v0, v2, s[0:1] offset:2 dlc
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll
index 9c9d6e01e5c06..65e8c1e6d0fc9 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll
@@ -148,18 +148,12 @@ define <4 x half> @v_constained_fma_v4f16_fpexcept_strict(<4 x half> %x, <4 x ha
; GFX10-LABEL: v_constained_fma_v4f16_fpexcept_strict:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v11, 16, v0
+; GFX10-NEXT: v_fma_f16 v6, v1, v3, v5 op_sel:[1,1,1,0]
+; GFX10-NEXT: v_fma_f16 v7, v0, v2, v4 op_sel:[1,1,1,0]
; GFX10-NEXT: v_fmac_f16_e32 v4, v0, v2
-; GFX10-NEXT: v_fmac_f16_e32 v6, v8, v7
; GFX10-NEXT: v_fmac_f16_e32 v5, v1, v3
-; GFX10-NEXT: v_fmac_f16_e32 v9, v11, v10
+; GFX10-NEXT: v_perm_b32 v0, v7, v4, 0x5040100
; GFX10-NEXT: v_perm_b32 v1, v6, v5, 0x5040100
-; GFX10-NEXT: v_perm_b32 v0, v9, v4, 0x5040100
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_constained_fma_v4f16_fpexcept_strict:
@@ -175,18 +169,12 @@ define <4 x half> @v_constained_fma_v4f16_fpexcept_strict(<4 x half> %x, <4 x ha
; GFX11-FAKE16-LABEL: v_constained_fma_v4f16_fpexcept_strict:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v4
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v0
+; GFX11-FAKE16-NEXT: v_fma_f16 v6, v1, v3, v5 op_sel:[1,1,1,0]
+; GFX11-FAKE16-NEXT: v_fma_f16 v7, v0, v2, v4 op_sel:[1,1,1,0]
; GFX11-FAKE16-NEXT: v_fmac_f16_e32 v4, v0, v2
-; GFX11-FAKE16-NEXT: v_fmac_f16_e32 v6, v8, v7
; GFX11-FAKE16-NEXT: v_fmac_f16_e32 v5, v1, v3
-; GFX11-FAKE16-NEXT: v_fmac_f16_e32 v9, v11, v10
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v7, v4, 0x5040100
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v6, v5, 0x5040100
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v9, v4, 0x5040100
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> %x, <4 x half> %y, <4 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
ret <4 x half> %val
>From 86360c1e747767fd4d9f1477b4e933556ec1ad36 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Thu, 12 Nov 2026 23:05:37 +0100
Subject: [PATCH 2/3] format
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index eb8e342f2c33b..7597f9c86f0ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3680,8 +3680,7 @@ bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
return true;
}
-bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In,
- SDValue &Src) const {
+bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In, SDValue &Src) const {
SDValue Unused;
if (!Subtarget->useRealTrue16Insts() && In.getValueSizeInBits() == 16 &&
isExtractHiElt(In, Unused))
>From 03df4e980e0940f569fbba1681ebba8ac6f7e729 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 13 Nov 2026 00:40:44 +0100
Subject: [PATCH 3/3] apply review
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 3 +--
llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp | 5 +----
llvm/lib/Target/AMDGPU/SIInstructions.td | 6 +++---
3 files changed, 5 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 7597f9c86f0ec..5b436491cac3f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3682,8 +3682,7 @@ bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In, SDValue &Src) const {
SDValue Unused;
- if (!Subtarget->useRealTrue16Insts() && In.getValueSizeInBits() == 16 &&
- isExtractHiElt(In, Unused))
+ if (isExtractHiElt(In, Unused))
return false;
return SelectVOP3NoMods(In, Src);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index c7af406dab056..4f444b2a8735b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -4950,11 +4950,8 @@ AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3NoOpSelMods(MachineOperand &Root) const {
- Register Reg = Root.getReg();
Register Unused;
- if (!Subtarget->useRealTrue16Insts() &&
- MRI->getType(Reg).getSizeInBits() == 16 &&
- isExtractHiElt(*MRI, Reg, Unused))
+ if (isExtractHiElt(*MRI, Root.getReg(), Unused))
return {};
return selectVOP3NoMods(Root);
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 4bd62c6c2107e..916baa0a978ab 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3953,9 +3953,9 @@ def : GCNPat <
>;
let True16Predicate = UseRealTrue16Insts in
def : GCNPat <
- (fma (f16 (VOP3NoOpSelMods f16:$src0)),
- (f16 (VOP3NoOpSelMods f16:$src1)),
- (f16 (VOP3NoOpSelMods f16:$src2))),
+ (fma (f16 (VOP3NoMods f16:$src0)),
+ (f16 (VOP3NoMods f16:$src1)),
+ (f16 (VOP3NoMods f16:$src2))),
(V_FMAC_F16_t16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,
SRCMODS.NONE, $src2)
>;
More information about the llvm-branch-commits
mailing list