[llvm-branch-commits] [llvm] [AMDGPU] Fold fpround of fadd and fsub into v_mad/fma_mixlo and mixhi (PR #224913)
Dmitry Sidorov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Sep 20 15:12:17 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/224913
>From 1c0c2b720c2bef9b45d8f3e0a9b0c0ba1cec1c03 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 19 Sep 2026 02:37:11 +0200
Subject: [PATCH] [AMDGPU] Fold fpround of fadd and fsub into v_mad/fma_mixlo
and mixhi
MadFmaMixFP32Pats turns (fadd x, y) into (fma x, 1.0, y) and (fsub x, y)
into (fma (-y), 1.0, x) so the mix instructions absorb the operation along
with the f16 or bf16 source modifiers. MadFmaMixFP16Pats and
MadFmaMixFP16Pats_t16 only did this for fmul, so a rounded result still
needed a separate convert for a rounding the mix instructions perform
themselves.
Unlike the f32 patterns these do not require an operand to be an fpextend
of an f16, since an fpround on the result always removes the convert. The
rewrite is exact because the mix instructions round the f32 result again
when they write the 16-bit destination, so it stays f32_to_f16(fma(x, 1.0,
y)).
Assisted-by: Claude Code Opus 5
---
llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 70 +++
.../CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll | 477 +++++++-----------
llvm/test/CodeGen/AMDGPU/bf16.ll | 14 +-
llvm/test/CodeGen/AMDGPU/fdiv.f16.ll | 11 +-
.../AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll | 24 +-
llvm/test/CodeGen/AMDGPU/frem.ll | 78 +--
llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll | 37 +-
llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll | 207 +++-----
llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll | 91 ++--
llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll | 372 +++-----------
10 files changed, 496 insertions(+), 885 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index d086b59001b1e..7d73bb328c590 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -300,12 +300,17 @@ multiclass MadFmaMixFP32Pats<SDPatternOperator fma_like,
DSTCLAMP.NONE)>;
}
+// The mix instructions round the f32 result again when they write the f16 or
+// bf16 destination, so these folds match a separate convert exactly.
multiclass MadFmaMixFP16Pats<SDPatternOperator fma_like,
Instruction mixlo_inst,
Instruction mixhi_inst,
ValueType VT = f16,
ValueType vecVT = v2f16> {
defvar VOP3PMadMixModsPat = !if (!eq(VT, bf16), VOP3PMadMixBF16Mods, VOP3PMadMixMods);
+ defvar VOP3PMadMixModsNegPat = !if (!eq(VT, bf16), VOP3PMadMixBF16ModsNeg, VOP3PMadMixModsNeg);
+ defvar OneImm = !if (!eq(VT, bf16), CONST.BF16_ONE, CONST.FP16_ONE);
+ defvar ImmMods = !if (!eq(VT, bf16), !or(SRCMODS.OP_SEL_0, SRCMODS.OP_SEL_1), SRCMODS.OP_SEL_1);
def : GCNPat <
(AMDGPUclamp (build_vector
(VT (fpround (fma_like (f32 (VOP3PMadMixModsPat VT:$lo_src0, i32:$lo_src0_modifiers)),
@@ -345,6 +350,48 @@ multiclass MadFmaMixFP16Pats<SDPatternOperator fma_like,
VGPR_32:$elt0))
>;
+ // (fadd x, y) -> (fma x, 1.0, y)
+ def : GCNPat <
+ (VT (fpround (fadd (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsPat f32:$src1, i32:$src1_modifiers))))),
+ (mixlo_inst $src0_modifiers, $src0,
+ (i32 ImmMods), (i32 OneImm),
+ $src1_modifiers, $src1,
+ DSTCLAMP.NONE,
+ (i32 (IMPLICIT_DEF)))
+ >;
+
+ def : GCNPat <
+ (build_vector VT:$elt0, (VT (fpround (fadd (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsPat f32:$src1, i32:$src1_modifiers)))))),
+ (vecVT (mixhi_inst $src0_modifiers, $src0,
+ (i32 ImmMods), (i32 OneImm),
+ $src1_modifiers, $src1,
+ DSTCLAMP.NONE,
+ VGPR_32:$elt0))
+ >;
+
+ // (fsub x, y) -> (fma (-y), 1.0, x)
+ def : GCNPat <
+ (VT (fpround (fsub (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsNegPat f32:$src1, i32:$src1_modifiers))))),
+ (mixlo_inst $src1_modifiers, $src1,
+ (i32 ImmMods), (i32 OneImm),
+ $src0_modifiers, $src0,
+ DSTCLAMP.NONE,
+ (i32 (IMPLICIT_DEF)))
+ >;
+
+ def : GCNPat <
+ (build_vector VT:$elt0, (VT (fpround (fsub (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsNegPat f32:$src1, i32:$src1_modifiers)))))),
+ (vecVT (mixhi_inst $src1_modifiers, $src1,
+ (i32 ImmMods), (i32 OneImm),
+ $src0_modifiers, $src0,
+ DSTCLAMP.NONE,
+ VGPR_32:$elt0))
+ >;
+
def : GCNPat <
(VT (fpround (fma_like (f32 (VOP3PMadMixModsPat VT:$src0, i32:$src0_modifiers)),
(f32 (VOP3PMadMixModsPat VT:$src1, i32:$src1_modifiers)),
@@ -389,6 +436,9 @@ multiclass MadFmaMixFP16Pats_t16<SDPatternOperator fma_like,
ValueType VT = f16,
ValueType vecVT = v2f16> {
defvar VOP3PMadMixModsPat = !if (!eq(VT, bf16), VOP3PMadMixBF16Mods, VOP3PMadMixMods);
+ defvar VOP3PMadMixModsNegPat = !if (!eq(VT, bf16), VOP3PMadMixBF16ModsNeg, VOP3PMadMixModsNeg);
+ defvar OneImm = !if (!eq(VT, bf16), CONST.BF16_ONE, CONST.FP16_ONE);
+ defvar ImmMods = !if (!eq(VT, bf16), !or(SRCMODS.OP_SEL_0, SRCMODS.OP_SEL_1), SRCMODS.OP_SEL_1);
def : GCNPat <
(VT (fpround (fmul (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
(f32 (VOP3PMadMixModsPat f32:$src1, i32:$src1_modifiers))))),
@@ -398,6 +448,26 @@ multiclass MadFmaMixFP16Pats_t16<SDPatternOperator fma_like,
DSTCLAMP.NONE)
>;
+ // (fadd x, y) -> (fma x, 1.0, y)
+ def : GCNPat <
+ (VT (fpround (fadd (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsPat f32:$src1, i32:$src1_modifiers))))),
+ (mix_inst_16 $src0_modifiers, $src0,
+ (i32 ImmMods), (i32 OneImm),
+ $src1_modifiers, $src1,
+ DSTCLAMP.NONE)
+ >;
+
+ // (fsub x, y) -> (fma (-y), 1.0, x)
+ def : GCNPat <
+ (VT (fpround (fsub (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsNegPat f32:$src1, i32:$src1_modifiers))))),
+ (mix_inst_16 $src1_modifiers, $src1,
+ (i32 ImmMods), (i32 OneImm),
+ $src0_modifiers, $src0,
+ DSTCLAMP.NONE)
+ >;
+
def : GCNPat <
(VT (fpround (fma_like (f32 (VOP3PMadMixModsPat VT:$src0, i32:$src0_modifiers)),
(f32 (VOP3PMadMixModsPat VT:$src1, i32:$src1_modifiers)),
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
index a42e915a4b479..6a76113ac3817 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
@@ -125,8 +125,7 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v1, v3, v0 op_sel_hi:[1,0,1]
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
@@ -144,8 +143,7 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
;
@@ -161,8 +159,7 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v4
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v3
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
@@ -178,8 +175,7 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX11-NEXT: v_fma_mix_f32 v4, -v1, v3, v0 op_sel_hi:[1,0,1]
; GFX11-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX11-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX11-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv half %a, %b
@@ -332,8 +328,7 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v1, v3, v0 op_sel_hi:[1,0,1]
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
@@ -351,8 +346,7 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
;
@@ -368,8 +362,7 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v4
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v3
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
@@ -385,8 +378,7 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX11-NEXT: v_fma_mix_f32 v4, -v1, v3, v0 op_sel_hi:[1,0,1]
; GFX11-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX11-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX11-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv half %a, %b
@@ -942,28 +934,26 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v3
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v0, v2, neg(0) op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v2
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v8, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v0, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v6, v0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v6, v7, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v6, v0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v7, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v7, v6, v4
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v4, v5, v4
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v4, v4, v7
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v3, v5
-; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v8, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v8, v6, v4
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v2, v2, v1, v0
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v0, -v1, v8, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, v0, 1.0, v8 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v3, v5
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-IEEE-LABEL: v_fdiv_v2f16:
@@ -989,16 +979,14 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, v0, 1.0, v7 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, v0, 1.0, v8 op_sel:[1,0,0] op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v7, 16, v0
; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v8, v4
; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX10-IEEE-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX10-IEEE-NEXT: v_add_f32_e32 v4, v4, v6
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v4, v4, 1.0, v6 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v3, v1, v0
-; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v4, v2, v5
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v4, v2, v7
; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
;
@@ -1025,10 +1013,8 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v4, v4, v6
; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX10-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v9
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v4, v4, v11
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v9 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v4, v4, 1.0, v11 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v1, v0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v2, v7
; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -1050,15 +1036,14 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX11-NEXT: v_dual_fmac_f32 v5, v7, v3 :: v_dual_fmac_f32 v6, v8, v4
; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel_hi:[1,0,1]
; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_dual_mul_f32 v3, v7, v3 :: v_dual_mul_f32 v4, v8, v4
+; GFX11-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-NEXT: v_dual_mul_f32 v4, v8, v4 :: v_dual_and_b32 v3, 0xff800000, v3
; GFX11-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX11-NEXT: v_dual_add_f32 v4, v4, v6 :: v_dual_and_b32 v3, 0xff800000, v3
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX11-NEXT: v_fma_mixlo_f16 v4, v4, 1.0, v6 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v3, v1, v0
-; GFX11-NEXT: v_div_fixup_f16 v1, v4, v2, v5
+; GFX11-NEXT: v_div_fixup_f16 v1, v4, v2, v7
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv <2 x half> %a, %b
@@ -1337,28 +1322,26 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v3
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v0, v2, neg(0) op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v2
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v8, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v0, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v6, v0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v6, v7, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v6, v0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v7, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v7, v6, v4
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v4, v5, v4
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v4, v4, v7
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v3, v5
-; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v8, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v8, v6, v4
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v2, v2, v1, v0
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v0, -v1, v8, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, v0, 1.0, v8 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v3, v5
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-IEEE-LABEL: v_fdiv_v2f16_ulp25:
@@ -1384,16 +1367,14 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, v0, 1.0, v7 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, v0, 1.0, v8 op_sel:[1,0,0] op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v7, 16, v0
; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v8, v4
; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX10-IEEE-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX10-IEEE-NEXT: v_add_f32_e32 v4, v4, v6
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v4, v4, 1.0, v6 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v3, v1, v0
-; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v4, v2, v5
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v4, v2, v7
; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
;
@@ -1420,10 +1401,8 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v4, v4, v6
; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX10-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v9
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v4, v4, v11
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v9 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v4, v4, 1.0, v11 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v1, v0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v2, v7
; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -1445,15 +1424,14 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX11-NEXT: v_dual_fmac_f32 v5, v7, v3 :: v_dual_fmac_f32 v6, v8, v4
; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel_hi:[1,0,1]
; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_dual_mul_f32 v3, v7, v3 :: v_dual_mul_f32 v4, v8, v4
+; GFX11-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-NEXT: v_dual_mul_f32 v4, v8, v4 :: v_dual_and_b32 v3, 0xff800000, v3
; GFX11-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX11-NEXT: v_dual_add_f32 v4, v4, v6 :: v_dual_and_b32 v3, 0xff800000, v3
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX11-NEXT: v_fma_mixlo_f16 v4, v4, 1.0, v6 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v3, v1, v0
-; GFX11-NEXT: v_div_fixup_f16 v1, v4, v2, v5
+; GFX11-NEXT: v_div_fixup_f16 v1, v4, v2, v7
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv <2 x half> %a, %b
@@ -1638,29 +1616,27 @@ define <2 x half> @v_rcp_v2f16(<2 x half> %x) {
; GFX9-FLUSH-LABEL: v_rcp_v2f16:
; GFX9-FLUSH: ; %bb.0:
; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX9-FLUSH-NEXT: v_mov_b32_e32 v4, 1.0
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v0, v1, v4 op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_mad_f32 v5, v5, v1, v1
-; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, v3
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v1, v7, v1
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v1, v1, v5
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v6
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
-; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v1, v1, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v1, v0, 1.0
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v0, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, v0, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v2, 1.0
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-IEEE-LABEL: v_rcp_v2f16:
@@ -1687,10 +1663,8 @@ define <2 x half> @v_rcp_v2f16(<2 x half> %x) {
; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v4
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v4 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -1714,10 +1688,8 @@ define <2 x half> @v_rcp_v2f16(<2 x half> %x) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v7 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -1742,9 +1714,8 @@ define <2 x half> @v_rcp_v2f16(<2 x half> %x) {
; GFX11-NEXT: v_dual_mul_f32 v2, v7, v2 :: v_dual_mul_f32 v3, v4, v3
; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX11-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX11-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_add_f32 v3, v3, v6
-; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX11-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v6 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -1931,29 +1902,27 @@ define <2 x half> @v_neg_rcp_v2f16(<2 x half> %x) {
; GFX9-FLUSH-LABEL: v_neg_rcp_v2f16:
; GFX9-FLUSH: ; %bb.0:
; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX9-FLUSH-NEXT: v_mov_b32_e32 v4, -1.0
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v0, v1, v4 op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_mad_f32 v5, v5, v1, -v1
-; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, -v3
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v1, v7, v1
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v1, v1, v5
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v6
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, -1.0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v2, -1.0
-; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, -v3
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v1, v1, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v1, v0, -1.0
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v0, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, v0, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v2, -1.0
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-IEEE-LABEL: v_neg_rcp_v2f16:
@@ -1980,10 +1949,8 @@ define <2 x half> @v_neg_rcp_v2f16(<2 x half> %x) {
; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v4
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v4 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -2007,10 +1974,8 @@ define <2 x half> @v_neg_rcp_v2f16(<2 x half> %x) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v7 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -2035,9 +2000,8 @@ define <2 x half> @v_neg_rcp_v2f16(<2 x half> %x) {
; GFX11-NEXT: v_dual_mul_f32 v2, v7, v2 :: v_dual_mul_f32 v3, v4, v3
; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX11-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX11-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_add_f32 v3, v3, v6
-; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX11-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v6 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -2246,10 +2210,8 @@ define <2 x half> @v_rcp_v2f16_fabs(<2 x half> %x) {
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v4
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, v0, v7
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, v0, 1.0, v7 op_sel_hi:[0,1,0]
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v3, 1.0
; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
@@ -2280,10 +2242,8 @@ define <2 x half> @v_rcp_v2f16_fabs(<2 x half> %x) {
; GFX10-IEEE-NEXT: v_mul_f32_e32 v0, v0, v4
; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX10-IEEE-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-IEEE-NEXT: v_add_f32_e32 v0, v0, v6
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v6 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v0, v2, 1.0
; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
@@ -2308,10 +2268,8 @@ define <2 x half> @v_rcp_v2f16_fabs(<2 x half> %x) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v7 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -2322,7 +2280,9 @@ define <2 x half> @v_rcp_v2f16_fabs(<2 x half> %x) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
; GFX11-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX11-NEXT: v_rcp_f32_e32 v3, v3
; GFX11-NEXT: v_rcp_f32_e32 v4, v4
; GFX11-NEXT: v_mov_b32_e32 v5, 1.0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -2330,20 +2290,15 @@ define <2 x half> @v_rcp_v2f16_fabs(<2 x half> %x) {
; GFX11-NEXT: v_fma_f32 v7, v7, v4, v4
; GFX11-NEXT: v_fma_mix_f32 v0, -|v0|, v7, v5 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX11-NEXT: v_mul_f32_e32 v0, v0, v4
-; GFX11-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v7
-; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
-; GFX11-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX11-NEXT: v_rcp_f32_e32 v3, v3
-; GFX11-NEXT: v_div_fixup_f16 v0, v0, v2, 1.0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_fma_mix_f32 v6, -v1, v3, v5 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_and_b32_e32 v0, 0xff800000, v0
; GFX11-NEXT: v_fma_f32 v6, v6, v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v7 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v5 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_div_fixup_f16 v0, v0, v2, 1.0
; GFX11-NEXT: v_mul_f32_e32 v3, v8, v3
; GFX11-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v6
-; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v6 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX11-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -2552,10 +2507,8 @@ define <2 x half> @v_neg_rcp_v2f16_fabs(<2 x half> %x) {
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v4
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, v0, v7
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, v0, 1.0, v7 op_sel_hi:[0,1,0]
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v3, -1.0
; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
@@ -2586,10 +2539,8 @@ define <2 x half> @v_neg_rcp_v2f16_fabs(<2 x half> %x) {
; GFX10-IEEE-NEXT: v_mul_f32_e32 v0, v0, v4
; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX10-IEEE-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-IEEE-NEXT: v_add_f32_e32 v0, v0, v6
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v6 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v0, v2, -1.0
; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
@@ -2614,10 +2565,8 @@ define <2 x half> @v_neg_rcp_v2f16_fabs(<2 x half> %x) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v7 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -2628,7 +2577,9 @@ define <2 x half> @v_neg_rcp_v2f16_fabs(<2 x half> %x) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
; GFX11-NEXT: v_cvt_f32_f16_e32 v4, v2
+; GFX11-NEXT: v_rcp_f32_e32 v3, v3
; GFX11-NEXT: v_rcp_f32_e32 v4, v4
; GFX11-NEXT: v_mov_b32_e32 v5, -1.0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -2636,20 +2587,15 @@ define <2 x half> @v_neg_rcp_v2f16_fabs(<2 x half> %x) {
; GFX11-NEXT: v_fma_f32 v7, v7, v4, -v4
; GFX11-NEXT: v_fma_mix_f32 v0, -|v0|, v7, v5 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX11-NEXT: v_mul_f32_e32 v0, v0, v4
-; GFX11-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v7
-; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v1
-; GFX11-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX11-NEXT: v_rcp_f32_e32 v3, v3
-; GFX11-NEXT: v_div_fixup_f16 v0, v0, v2, -1.0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_fma_mix_f32 v6, v1, v3, v5 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_and_b32_e32 v0, 0xff800000, v0
; GFX11-NEXT: v_fma_f32 v6, v6, v3, -v3
+; GFX11-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v7 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v5 op_sel_hi:[1,0,0]
+; GFX11-NEXT: v_div_fixup_f16 v0, v0, v2, -1.0
; GFX11-NEXT: v_mul_f32_e32 v3, v8, v3
; GFX11-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v6
-; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v6 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX11-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -3009,29 +2955,27 @@ define <2 x half> @v_rcp_v2f16_ulp25(<2 x half> %x) {
; GFX9-FLUSH-LABEL: v_rcp_v2f16_ulp25:
; GFX9-FLUSH: ; %bb.0:
; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX9-FLUSH-NEXT: v_mov_b32_e32 v4, 1.0
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v0, v1, v4 op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_mad_f32 v5, v5, v1, v1
-; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, v3
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v0, v5, v4 op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v1, v7, v1
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v0, v3, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v1, v1, v5
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v6
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
-; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-FLUSH-NEXT: v_mad_f32 v6, v6, v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v1, v1, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v1, v0, 1.0
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v0, -v0, v6, v4 op_sel:[1,0,0] op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, v0, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v2, 1.0
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-IEEE-LABEL: v_rcp_v2f16_ulp25:
@@ -3058,10 +3002,8 @@ define <2 x half> @v_rcp_v2f16_ulp25(<2 x half> %x) {
; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v4
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v4 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -3085,10 +3027,8 @@ define <2 x half> @v_rcp_v2f16_ulp25(<2 x half> %x) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v7 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -3113,9 +3053,8 @@ define <2 x half> @v_rcp_v2f16_ulp25(<2 x half> %x) {
; GFX11-NEXT: v_dual_mul_f32 v2, v7, v2 :: v_dual_mul_f32 v3, v4, v3
; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX11-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX11-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_add_f32 v3, v3, v6
-; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX11-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v6 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -3517,8 +3456,7 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v3, v0
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
; GFX9-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, s1, v2
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, s1, 1.0, v2 op_sel_hi:[0,1,0]
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v1, s0
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-FLUSH-NEXT: ; return to shader part epilog
@@ -3537,8 +3475,7 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX10-IEEE-NEXT: v_mul_f32_e32 v0, v2, v0
; GFX10-IEEE-NEXT: v_readfirstlane_b32 s2, v0
; GFX10-IEEE-NEXT: s_and_b32 s2, s2, 0xff800000
-; GFX10-IEEE-NEXT: v_add_f32_e32 v0, s2, v1
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v0, s2, 1.0, v1 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v0, s1, s0
; GFX10-IEEE-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-IEEE-NEXT: ; return to shader part epilog
@@ -3555,8 +3492,7 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s2, v0
; GFX10-FLUSH-NEXT: s_and_b32 s2, s2, 0xff800000
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v0, s2, v3
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v0, s2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v0, s1, s0
; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-FLUSH-NEXT: ; return to shader part epilog
@@ -3573,8 +3509,7 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX11-NEXT: v_mul_f32_e32 v0, v2, v0
; GFX11-NEXT: v_readfirstlane_b32 s2, v0
; GFX11-NEXT: s_and_b32 s2, s2, 0xff800000
-; GFX11-NEXT: v_add_f32_e32 v0, s2, v1
-; GFX11-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX11-NEXT: v_fma_mixlo_f16 v0, s2, 1.0, v1 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v0, s1, s0
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
@@ -3925,8 +3860,8 @@ define amdgpu_ps i32 @s_fdiv_v2f16(i32 inreg %a.arg, i32 inreg %b.arg) {
; GFX9-FLUSH: ; %bb.0:
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
; GFX9-FLUSH-NEXT: s_lshr_b32 s2, s1, 16
-; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s2
; GFX9-FLUSH-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s2
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-FLUSH-NEXT: s_lshr_b32 s3, s0, 16
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v1, v1
@@ -3937,21 +3872,19 @@ define amdgpu_ps i32 @s_fdiv_v2f16(i32 inreg %a.arg, i32 inreg %b.arg) {
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v4, v0
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
; GFX9-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, s1, v3
-; GFX9-FLUSH-NEXT: v_mov_b32_e32 v3, s2
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, s3, v1, neg(0) op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v3, v4, s3 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v4, v5, v1
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v3, v4, s3 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v1, v5, v1
-; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v1
-; GFX9-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v1, s1, v4
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, s1, 1.0, v3 op_sel_hi:[0,1,0]
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v2, s0
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v3, s3
+; GFX9-FLUSH-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v2, s3, v1, neg(0) op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v3, -v0, v2, s3 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v2, v3, v1
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v3, -v0, v2, s3 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v1, s1, 1.0, v2 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, s3
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
; GFX9-FLUSH-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-FLUSH-NEXT: ; return to shader part epilog
@@ -3984,10 +3917,8 @@ define amdgpu_ps i32 @s_fdiv_v2f16(i32 inreg %a.arg, i32 inreg %b.arg) {
; GFX10-IEEE-NEXT: v_readfirstlane_b32 s5, v1
; GFX10-IEEE-NEXT: s_and_b32 s4, s4, 0xff800000
; GFX10-IEEE-NEXT: s_and_b32 s5, s5, 0xff800000
-; GFX10-IEEE-NEXT: v_add_f32_e32 v0, s4, v2
-; GFX10-IEEE-NEXT: v_add_f32_e32 v1, s5, v3
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v0, s4, 1.0, v2 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v1, s5, 1.0, v3 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v0, s1, s0
; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v1, s2, s3
; GFX10-IEEE-NEXT: v_readfirstlane_b32 s0, v0
@@ -4019,10 +3950,8 @@ define amdgpu_ps i32 @s_fdiv_v2f16(i32 inreg %a.arg, i32 inreg %b.arg) {
; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s5, v1
; GFX10-FLUSH-NEXT: s_and_b32 s4, s4, 0xff800000
; GFX10-FLUSH-NEXT: s_and_b32 s5, s5, 0xff800000
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v0, s4, v5
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v1, s5, v7
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v0, s4, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v1, s5, 1.0, v7 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v0, s1, s0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v1, s2, s3
; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
@@ -4051,9 +3980,8 @@ define amdgpu_ps i32 @s_fdiv_v2f16(i32 inreg %a.arg, i32 inreg %b.arg) {
; GFX11-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-NEXT: s_and_b32 s4, s4, 0xff800000
; GFX11-NEXT: s_and_b32 s5, s5, 0xff800000
-; GFX11-NEXT: v_dual_add_f32 v0, s4, v2 :: v_dual_add_f32 v1, s5, v3
-; GFX11-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX11-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX11-NEXT: v_fma_mixlo_f16 v0, s4, 1.0, v2 op_sel_hi:[0,1,0]
+; GFX11-NEXT: v_fma_mixlo_f16 v1, s5, 1.0, v3 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v0, s1, s0
; GFX11-NEXT: v_div_fixup_f16 v1, v1, s2, s3
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@@ -4506,15 +4434,13 @@ define amdgpu_ps i32 @s_rsq_v2f16(i32 inreg %a.arg) {
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s0, v2
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v4, v3
; GFX9-FLUSH-NEXT: s_and_b32 s0, s0, 0xff800000
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, s0, v5
-; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s0, v2
-; GFX9-FLUSH-NEXT: s_and_b32 s0, s0, 0xff800000
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, s0, v6
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v3, s0, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v2
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
+; GFX9-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, -1.0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, s1, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v1, -1.0
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
; GFX9-FLUSH-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX9-FLUSH-NEXT: ; return to shader part epilog
@@ -4541,10 +4467,8 @@ define amdgpu_ps i32 @s_rsq_v2f16(i32 inreg %a.arg) {
; GFX10-IEEE-NEXT: v_readfirstlane_b32 s1, v3
; GFX10-IEEE-NEXT: s_and_b32 s0, s0, 0xff800000
; GFX10-IEEE-NEXT: s_and_b32 s1, s1, 0xff800000
-; GFX10-IEEE-NEXT: v_add_f32_e32 v2, s0, v5
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, s1, v6
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v2, s0, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, s1, 1.0, v6 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX10-IEEE-NEXT: v_readfirstlane_b32 s0, v0
@@ -4573,10 +4497,8 @@ define amdgpu_ps i32 @s_rsq_v2f16(i32 inreg %a.arg) {
; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s1, v3
; GFX10-FLUSH-NEXT: s_and_b32 s0, s0, 0xff800000
; GFX10-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, s0, v6
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, s1, v7
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, s0, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, s1, 1.0, v7 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
@@ -4607,9 +4529,8 @@ define amdgpu_ps i32 @s_rsq_v2f16(i32 inreg %a.arg) {
; GFX11-NEXT: v_readfirstlane_b32 s1, v3
; GFX11-NEXT: s_and_b32 s0, s0, 0xff800000
; GFX11-NEXT: s_and_b32 s1, s1, 0xff800000
-; GFX11-NEXT: v_dual_add_f32 v2, s0, v5 :: v_dual_add_f32 v3, s1, v6
-; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v2, s0, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX11-NEXT: v_fma_mixlo_f16 v3, s1, 1.0, v6 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@@ -5600,12 +5521,10 @@ define <2 x half> @v_rsq_v2f16(<2 x half> %a) {
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v4, v3
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v6
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, 1.0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v3, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
@@ -5629,10 +5548,8 @@ define <2 x half> @v_rsq_v2f16(<2 x half> %a) {
; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v4, v3
; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v6
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v6 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v3, v0, 1.0
; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
@@ -5657,10 +5574,8 @@ define <2 x half> @v_rsq_v2f16(<2 x half> %a) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v7 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, 1.0
; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
@@ -5688,9 +5603,8 @@ define <2 x half> @v_rsq_v2f16(<2 x half> %a) {
; GFX11-NEXT: v_dual_mul_f32 v2, v7, v2 :: v_dual_mul_f32 v3, v4, v3
; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX11-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX11-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_add_f32 v3, v3, v6
-; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX11-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v6 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, 1.0
; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, 1.0
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -5905,12 +5819,10 @@ define <2 x half> @v_neg_rsq_v2f16(<2 x half> %a) {
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v3, v4, v3
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX9-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v3, v3, v6
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v3, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
@@ -5934,10 +5846,8 @@ define <2 x half> @v_neg_rsq_v2f16(<2 x half> %a) {
; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v4, v3
; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v6
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v6 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
@@ -5962,10 +5872,8 @@ define <2 x half> @v_neg_rsq_v2f16(<2 x half> %a) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v7 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
@@ -5993,9 +5901,8 @@ define <2 x half> @v_neg_rsq_v2f16(<2 x half> %a) {
; GFX11-NEXT: v_dual_mul_f32 v2, v7, v2 :: v_dual_mul_f32 v3, v4, v3
; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX11-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX11-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_add_f32 v3, v3, v6
-; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX11-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v6 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
; GFX11-NEXT: v_div_fixup_f16 v1, v3, v1, -1.0
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 04fe0d297b84d..34bd4297036b3 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -29498,9 +29498,8 @@ define bfloat @v_log_bf16(bfloat %a) #0 {
; GFX1250-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1250-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_fma_mixlo_bf16 v0, -v1, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.log.bf16(bfloat %a)
ret bfloat %op
@@ -29921,9 +29920,8 @@ define bfloat @v_log10_bf16(bfloat %a) #0 {
; GFX1250-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1250-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_fma_mixlo_bf16 v0, -v1, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.log10.bf16(bfloat %a)
ret bfloat %op
@@ -31310,9 +31308,7 @@ define bfloat @v_round_bf16(bfloat %a) #0 {
; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
-; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.round.bf16(bfloat %a)
ret bfloat %op
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll b/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
index 6f2bf9a4baa07..06759f3979c13 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
@@ -123,8 +123,7 @@ define amdgpu_kernel void @v_fdiv_f16(
; GFX10-NEXT: v_mad_f32 v3, -v3, v5, v6
; GFX10-NEXT: v_mul_f32_e32 v3, v3, v4
; GFX10-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
; GFX10-NEXT: v_div_fixup_f16 v1, v3, v2, v1
; GFX10-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
@@ -155,9 +154,7 @@ define amdgpu_kernel void @v_fdiv_f16(
; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v4 op_sel_hi:[0,1,0]
; GFX11-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
@@ -188,9 +185,7 @@ define amdgpu_kernel void @v_fdiv_f16(
; GFX11-FAKE16-NEXT: v_mul_f32_e32 v3, v5, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v4 op_sel_hi:[0,1,0]
; GFX11-FAKE16-NEXT: v_div_fixup_f16 v1, v3, v2, v1
; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
index 8ddab5583b9ee..e24567054ce38 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
@@ -1547,8 +1547,7 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind {
; GFX10-NEXT: v_mad_f32 v1, -v1, v3, 0x46000000
; GFX10-NEXT: v_mul_f32_e32 v1, v1, v2
; GFX10-NEXT: v_and_b32_e32 v1, 0xff800000, v1
-; GFX10-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX10-NEXT: v_fma_mixlo_f16 v1, v1, 1.0, v3 op_sel_hi:[0,1,0]
; GFX10-NEXT: v_div_fixup_f16 v0, v1, v0, 0x7000
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -1573,9 +1572,8 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind {
; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, v3, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
+; GFX11-TRUE16-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v2 op_sel_hi:[0,1,0]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v1.l, 0x7000
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1600,9 +1598,8 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind {
; GFX11-FAKE16-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff800000, v1
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX11-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, 1.0, v2 op_sel_hi:[0,1,0]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_div_fixup_f16 v0, v1, v0, 0x7000
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%shl = shl nuw i32 1, %cnt
@@ -1715,8 +1712,7 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind {
; GFX10-NEXT: v_mad_f32 v1, -v1, v3, 2.0
; GFX10-NEXT: v_mul_f32_e32 v1, v1, v2
; GFX10-NEXT: v_and_b32_e32 v1, 0xff800000, v1
-; GFX10-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX10-NEXT: v_fma_mixlo_f16 v1, v1, 1.0, v3 op_sel_hi:[0,1,0]
; GFX10-NEXT: v_div_fixup_f16 v0, v1, v0, 2.0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -1740,9 +1736,7 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind {
; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, v3, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v2 op_sel_hi:[0,1,0]
; GFX11-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v1.l, 2.0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1766,9 +1760,7 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind {
; GFX11-FAKE16-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff800000, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_fma_mixlo_f16 v1, v1, 1.0, v2 op_sel_hi:[0,1,0]
; GFX11-FAKE16-NEXT: v_div_fixup_f16 v0, v1, v0, 2.0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%shl = shl nuw i16 1, %cnt
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 486d7c15150bc..19d98069ee468 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -1484,8 +1484,7 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX10-NEXT: v_mad_f32 v3, -v3, v5, v6
; GFX10-NEXT: v_mul_f32_e32 v3, v3, v4
; GFX10-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
; GFX10-NEXT: v_div_fixup_f16 v3, v3, v2, v1
; GFX10-NEXT: v_trunc_f16_e32 v3, v3
; GFX10-NEXT: v_fma_f16 v1, -v3, v2, v1
@@ -1516,12 +1515,10 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v4 op_sel_hi:[0,1,0]
; GFX11-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_fma_f16 v0.l, -v0.l, v3.l, v2.l
; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
@@ -1550,12 +1547,10 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX11-FAKE16-NEXT: v_mul_f32_e32 v3, v5, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v4 op_sel_hi:[0,1,0]
; GFX11-FAKE16-NEXT: v_div_fixup_f16 v3, v3, v2, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_trunc_f16_e32 v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_fma_f16 v1, -v3, v2, v1
; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
@@ -1583,14 +1578,12 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1150-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX1150-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX1150-TRUE16-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v4 op_sel_hi:[0,1,0]
; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1150-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
-; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
+; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-TRUE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
-; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1150-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v3.l
; GFX1150-TRUE16-NEXT: global_store_b16 v1, v2, s[0:1]
; GFX1150-TRUE16-NEXT: s_endpgm
@@ -1618,14 +1611,12 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1150-FAKE16-NEXT: v_mul_f32_e32 v3, v5, v3
; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-FAKE16-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX1150-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1150-FAKE16-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v4 op_sel_hi:[0,1,0]
; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-FAKE16-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX1150-FAKE16-NEXT: v_div_fixup_f16 v3, v3, v2, v1
-; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-FAKE16-NEXT: v_trunc_f16_e32 v3, v3
+; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-FAKE16-NEXT: v_xor_b32_e32 v3, 0x8000, v3
-; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1150-FAKE16-NEXT: v_fmac_f16_e32 v1, v3, v2
; GFX1150-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1150-FAKE16-NEXT: s_endpgm
@@ -1653,14 +1644,12 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX1200-TRUE16-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v4 op_sel_hi:[0,1,0]
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v3.l
; GFX1200-TRUE16-NEXT: global_store_b16 v1, v2, s[0:1]
; GFX1200-TRUE16-NEXT: s_endpgm
@@ -1688,14 +1677,12 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v3, v5, v3
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-FAKE16-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX1200-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1200-FAKE16-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v4 op_sel_hi:[0,1,0]
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX1200-FAKE16-NEXT: v_div_fixup_f16 v3, v3, v2, v1
-; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-FAKE16-NEXT: v_trunc_f16_e32 v3, v3
+; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v3, 0x8000, v3
-; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1200-FAKE16-NEXT: v_fmac_f16_e32 v1, v3, v2
; GFX1200-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1200-FAKE16-NEXT: s_endpgm
@@ -1967,8 +1954,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX10-NEXT: v_mad_f32 v3, -v3, v5, v6
; GFX10-NEXT: v_mul_f32_e32 v3, v3, v4
; GFX10-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX10-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX10-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
; GFX10-NEXT: v_div_fixup_f16 v3, v3, v2, v1
; GFX10-NEXT: v_trunc_f16_e32 v3, v3
; GFX10-NEXT: v_fma_f16 v1, -v3, v2, v1
@@ -1999,12 +1985,10 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v4 op_sel_hi:[0,1,0]
; GFX11-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_fma_f16 v0.l, -v0.l, v3.l, v2.l
; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
@@ -2033,12 +2017,10 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX11-FAKE16-NEXT: v_mul_f32_e32 v3, v5, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v4 op_sel_hi:[0,1,0]
; GFX11-FAKE16-NEXT: v_div_fixup_f16 v3, v3, v2, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_trunc_f16_e32 v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_fma_f16 v1, -v3, v2, v1
; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
@@ -2066,14 +2048,12 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1150-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX1150-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX1150-TRUE16-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v4 op_sel_hi:[0,1,0]
; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1150-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
-; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
+; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-TRUE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
-; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1150-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v3.l
; GFX1150-TRUE16-NEXT: global_store_b16 v1, v2, s[0:1]
; GFX1150-TRUE16-NEXT: s_endpgm
@@ -2101,14 +2081,12 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1150-FAKE16-NEXT: v_mul_f32_e32 v3, v5, v3
; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-FAKE16-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX1150-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1150-FAKE16-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v4 op_sel_hi:[0,1,0]
; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-FAKE16-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX1150-FAKE16-NEXT: v_div_fixup_f16 v3, v3, v2, v1
-; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-FAKE16-NEXT: v_trunc_f16_e32 v3, v3
+; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-FAKE16-NEXT: v_xor_b32_e32 v3, 0x8000, v3
-; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1150-FAKE16-NEXT: v_fmac_f16_e32 v1, v3, v2
; GFX1150-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1150-FAKE16-NEXT: s_endpgm
@@ -2136,14 +2114,12 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX1200-TRUE16-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v4 op_sel_hi:[0,1,0]
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v3.l
; GFX1200-TRUE16-NEXT: global_store_b16 v1, v2, s[0:1]
; GFX1200-TRUE16-NEXT: s_endpgm
@@ -2171,14 +2147,12 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v3, v5, v3
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-FAKE16-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX1200-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1200-FAKE16-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v4 op_sel_hi:[0,1,0]
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX1200-FAKE16-NEXT: v_div_fixup_f16 v3, v3, v2, v1
-; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-FAKE16-NEXT: v_trunc_f16_e32 v3, v3
+; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v3, 0x8000, v3
-; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1200-FAKE16-NEXT: v_fmac_f16_e32 v1, v3, v2
; GFX1200-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1200-FAKE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
index ad9acc26dcba3..5d51481f8622d 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
@@ -227,21 +227,17 @@ define <2 x bfloat> @mixhi_fptrunc_fadd(float %a, float %b, bfloat %lo) #0 {
; GFX1250-FAKE16: ; %bb.0: ; %.entry
; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
+; GFX1250-FAKE16-NEXT: v_fma_mixhi_bf16 v2, v0, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[0,1,0]
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-REAL16-LABEL: mixhi_fptrunc_fadd:
; GFX1250-REAL16: ; %bb.0: ; %.entry
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
+; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v0, v0, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[0,1,0]
; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
.entry:
%add = fadd float %a, %b
@@ -256,21 +252,20 @@ define <2 x bfloat> @mixhi_fptrunc_fadd_f16_src(half %a, float %b, bfloat %lo) #
; GFX1250-FAKE16: ; %bb.0: ; %.entry
; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
+; GFX1250-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
+; GFX1250-FAKE16-NEXT: v_fma_mixhi_bf16 v2, v0, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[0,1,0]
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-REAL16-LABEL: mixhi_fptrunc_fadd_f16_src:
; GFX1250-REAL16: ; %bb.0: ; %.entry
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
+; GFX1250-REAL16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v0, v0, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[0,1,0]
; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
.entry:
%a.ext = fpext half %a to float
@@ -286,21 +281,17 @@ define <2 x bfloat> @mixhi_fptrunc_fsub(float %a, float %b, bfloat %lo) #0 {
; GFX1250-FAKE16: ; %bb.0: ; %.entry
; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
+; GFX1250-FAKE16-NEXT: v_fma_mixhi_bf16 v2, -v1, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-REAL16-LABEL: mixhi_fptrunc_fsub:
; GFX1250-REAL16: ; %bb.0: ; %.entry
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
+; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v0, -v1, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
.entry:
%sub = fsub float %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
index e918222a8993c..b549d44eebb2d 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
@@ -842,29 +842,24 @@ define <2 x half> @mixhi_fptrunc_fadd(float %a, float %b, half %lo) #0 {
; SDAG-GFX11-TRUE16-LABEL: mixhi_fptrunc_fadd:
; SDAG-GFX11-TRUE16: ; %bb.0: ; %.entry
; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.h, v0
+; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, v0, 1.0, v1 op_sel_hi:[0,1,0]
; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX11-FAKE16-LABEL: mixhi_fptrunc_fadd:
; SDAG-GFX11-FAKE16: ; %bb.0: ; %.entry
; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; SDAG-GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
+; SDAG-GFX11-FAKE16-NEXT: v_fma_mixhi_f16 v2, v0, 1.0, v1 op_sel_hi:[0,1,0]
+; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v2
; SDAG-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; SDAG-GFX9-LABEL: mixhi_fptrunc_fadd:
-; SDAG-GFX9: ; %bb.0: ; %.entry
-; SDAG-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX9-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; SDAG-GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
-; SDAG-GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: mixhi_fptrunc_fadd:
+; GFX9: ; %bb.0: ; %.entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mad_mixhi_f16 v2, v0, 1.0, v1 op_sel_hi:[0,1,0]
+; GFX9-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-VI-LABEL: mixhi_fptrunc_fadd:
; SDAG-VI: ; %bb.0: ; %.entry
@@ -884,31 +879,13 @@ define <2 x half> @mixhi_fptrunc_fadd(float %a, float %b, half %lo) #0 {
; SDAG-CI-NEXT: v_or_b32_e32 v0, v1, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX11-TRUE16-LABEL: mixhi_fptrunc_fadd:
-; GISEL-GFX11-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v2.h, v0
-; GISEL-GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GISEL-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX11-FAKE16-LABEL: mixhi_fptrunc_fadd:
-; GISEL-GFX11-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-FAKE16-NEXT: v_dual_add_f32 v0, v0, v1 :: v_dual_and_b32 v1, 0xffff, v2
-; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GISEL-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX9-LABEL: mixhi_fptrunc_fadd:
-; GISEL-GFX9: ; %bb.0: ; %.entry
-; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX9-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-GFX9-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; GISEL-GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
+; GISEL-GFX11-LABEL: mixhi_fptrunc_fadd:
+; GISEL-GFX11: ; %bb.0: ; %.entry
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_fma_mixhi_f16 v2, v0, 1.0, v1 op_sel_hi:[0,1,0]
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, v2
+; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-VI-LABEL: mixhi_fptrunc_fadd:
; GISEL-VI: ; %bb.0: ; %.entry
@@ -942,19 +919,16 @@ define <2 x half> @mixhi_fptrunc_fadd_f16_denormals(float %a, float %b, half %lo
; SDAG-GFX11-TRUE16-LABEL: mixhi_fptrunc_fadd_f16_denormals:
; SDAG-GFX11-TRUE16: ; %bb.0: ; %.entry
; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.h, v0
+; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, v0, 1.0, v1 op_sel_hi:[0,1,0]
; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX11-FAKE16-LABEL: mixhi_fptrunc_fadd_f16_denormals:
; SDAG-GFX11-FAKE16: ; %bb.0: ; %.entry
; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; SDAG-GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
+; SDAG-GFX11-FAKE16-NEXT: v_fma_mixhi_f16 v2, v0, 1.0, v1 op_sel_hi:[0,1,0]
+; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v2
; SDAG-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX9-LABEL: mixhi_fptrunc_fadd_f16_denormals:
@@ -984,23 +958,13 @@ define <2 x half> @mixhi_fptrunc_fadd_f16_denormals(float %a, float %b, half %lo
; SDAG-CI-NEXT: v_or_b32_e32 v0, v1, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX11-TRUE16-LABEL: mixhi_fptrunc_fadd_f16_denormals:
-; GISEL-GFX11-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v2.h, v0
-; GISEL-GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GISEL-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX11-FAKE16-LABEL: mixhi_fptrunc_fadd_f16_denormals:
-; GISEL-GFX11-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-FAKE16-NEXT: v_dual_add_f32 v0, v0, v1 :: v_dual_and_b32 v1, 0xffff, v2
-; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GISEL-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GISEL-GFX11-LABEL: mixhi_fptrunc_fadd_f16_denormals:
+; GISEL-GFX11: ; %bb.0: ; %.entry
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_fma_mixhi_f16 v2, v0, 1.0, v1 op_sel_hi:[0,1,0]
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, v2
+; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX9-LABEL: mixhi_fptrunc_fadd_f16_denormals:
; GISEL-GFX9: ; %bb.0: ; %.entry
@@ -1042,29 +1006,24 @@ define <2 x half> @mixhi_fptrunc_fsub(float %a, float %b, half %lo) #0 {
; SDAG-GFX11-TRUE16-LABEL: mixhi_fptrunc_fsub:
; SDAG-GFX11-TRUE16: ; %bb.0: ; %.entry
; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; SDAG-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.h, v0
+; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, -v1, 1.0, v0 op_sel_hi:[0,1,0]
; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX11-FAKE16-LABEL: mixhi_fptrunc_fsub:
; SDAG-GFX11-FAKE16: ; %bb.0: ; %.entry
; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; SDAG-GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
+; SDAG-GFX11-FAKE16-NEXT: v_fma_mixhi_f16 v2, -v1, 1.0, v0 op_sel_hi:[0,1,0]
+; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v2
; SDAG-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; SDAG-GFX9-LABEL: mixhi_fptrunc_fsub:
-; SDAG-GFX9: ; %bb.0: ; %.entry
-; SDAG-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
-; SDAG-GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; SDAG-GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
-; SDAG-GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: mixhi_fptrunc_fsub:
+; GFX9: ; %bb.0: ; %.entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mad_mixhi_f16 v2, -v1, 1.0, v0 op_sel_hi:[0,1,0]
+; GFX9-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-VI-LABEL: mixhi_fptrunc_fsub:
; SDAG-VI: ; %bb.0: ; %.entry
@@ -1084,31 +1043,13 @@ define <2 x half> @mixhi_fptrunc_fsub(float %a, float %b, half %lo) #0 {
; SDAG-CI-NEXT: v_or_b32_e32 v0, v1, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX11-TRUE16-LABEL: mixhi_fptrunc_fsub:
-; GISEL-GFX11-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GISEL-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v2.h, v0
-; GISEL-GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GISEL-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX11-FAKE16-LABEL: mixhi_fptrunc_fsub:
-; GISEL-GFX11-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-FAKE16-NEXT: v_dual_sub_f32 v0, v0, v1 :: v_dual_and_b32 v1, 0xffff, v2
-; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GISEL-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX9-LABEL: mixhi_fptrunc_fsub:
-; GISEL-GFX9: ; %bb.0: ; %.entry
-; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
-; GISEL-GFX9-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; GISEL-GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
+; GISEL-GFX11-LABEL: mixhi_fptrunc_fsub:
+; GISEL-GFX11: ; %bb.0: ; %.entry
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_fma_mixhi_f16 v2, -v1, 1.0, v0 op_sel_hi:[0,1,0]
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, v2
+; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-VI-LABEL: mixhi_fptrunc_fsub:
; GISEL-VI: ; %bb.0: ; %.entry
@@ -1142,29 +1083,24 @@ define <2 x half> @mixhi_fptrunc_fadd_f16_src(half %a, float %b, half %lo) #0 {
; SDAG-GFX11-TRUE16-LABEL: mixhi_fptrunc_fadd_f16_src:
; SDAG-GFX11-TRUE16: ; %bb.0: ; %.entry
; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-TRUE16-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; SDAG-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.h, v0
+; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX11-FAKE16-LABEL: mixhi_fptrunc_fadd_f16_src:
; SDAG-GFX11-FAKE16: ; %bb.0: ; %.entry
; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-FAKE16-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; SDAG-GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
+; SDAG-GFX11-FAKE16-NEXT: v_fma_mixhi_f16 v2, v0, 1.0, v1 op_sel_hi:[1,1,0]
+; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v2
; SDAG-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; SDAG-GFX9-LABEL: mixhi_fptrunc_fadd_f16_src:
-; SDAG-GFX9: ; %bb.0: ; %.entry
-; SDAG-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX9-NEXT: v_mad_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; SDAG-GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; SDAG-GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
-; SDAG-GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: mixhi_fptrunc_fadd_f16_src:
+; GFX9: ; %bb.0: ; %.entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mad_mixhi_f16 v2, v0, 1.0, v1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-VI-LABEL: mixhi_fptrunc_fadd_f16_src:
; SDAG-VI: ; %bb.0: ; %.entry
@@ -1186,32 +1122,13 @@ define <2 x half> @mixhi_fptrunc_fadd_f16_src(half %a, float %b, half %lo) #0 {
; SDAG-CI-NEXT: v_or_b32_e32 v0, v1, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX11-TRUE16-LABEL: mixhi_fptrunc_fadd_f16_src:
-; GISEL-GFX11-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-TRUE16-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; GISEL-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v2.h, v0
-; GISEL-GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GISEL-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX11-FAKE16-LABEL: mixhi_fptrunc_fadd_f16_src:
-; GISEL-GFX11-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-FAKE16-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; GISEL-GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GISEL-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX9-LABEL: mixhi_fptrunc_fadd_f16_src:
-; GISEL-GFX9: ; %bb.0: ; %.entry
-; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX9-NEXT: v_mad_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; GISEL-GFX9-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; GISEL-GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
+; GISEL-GFX11-LABEL: mixhi_fptrunc_fadd_f16_src:
+; GISEL-GFX11: ; %bb.0: ; %.entry
+; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_fma_mixhi_f16 v2, v0, 1.0, v1 op_sel_hi:[1,1,0]
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, v2
+; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-VI-LABEL: mixhi_fptrunc_fadd_f16_src:
; GISEL-VI: ; %bb.0: ; %.entry
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll
index cb34b593a52dd..1d8c5e6743595 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll
@@ -664,9 +664,7 @@ define bfloat @mixlo_fptrunc_fadd(float %a, float %b) #0 {
; GFX1250: ; %bb.0: ; %.entry
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[0,1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
.entry:
%add = fadd float %a, %b
@@ -679,9 +677,7 @@ define bfloat @mixlo_fptrunc_fadd_abs_src_mod(float %a, float %b) #0 {
; GFX1250: ; %bb.0: ; %.entry
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_add_f32_e64 v0, |v0|, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_fma_mixlo_bf16 v0, |v0|, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[0,1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
.entry:
%a.fabs = call float @llvm.fabs.f32(float %a)
@@ -695,9 +691,7 @@ define bfloat @mixlo_fptrunc_fsub(float %a, float %b) #0 {
; GFX1250: ; %bb.0: ; %.entry
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_fma_mixlo_bf16 v0, -v1, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
.entry:
%sub = fsub float %a, %b
@@ -706,14 +700,23 @@ define bfloat @mixlo_fptrunc_fsub(float %a, float %b) #0 {
}
define bfloat @mixlo_fptrunc_fadd_f16_src(half %a, float %b) #0 {
-; GFX1250-LABEL: mixlo_fptrunc_fadd_f16_src:
-; GFX1250: ; %bb.0: ; %.entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-FAKE16-LABEL: mixlo_fptrunc_fadd_f16_src:
+; GFX1250-FAKE16: ; %bb.0: ; %.entry
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_fma_mixlo_bf16 v0, v0, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[0,1,0]
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-REAL16-LABEL: mixlo_fptrunc_fadd_f16_src:
+; GFX1250-REAL16: ; %bb.0: ; %.entry
+; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-REAL16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-REAL16-NEXT: v_fma_mixlo_bf16 v0, v0, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[0,1,0]
+; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
.entry:
%a.ext = fpext half %a to float
%add = fadd float %a.ext, %b
@@ -722,14 +725,23 @@ define bfloat @mixlo_fptrunc_fadd_f16_src(half %a, float %b) #0 {
}
define bfloat @mixlo_fptrunc_fsub_f16_src(half %a, float %b) #0 {
-; GFX1250-LABEL: mixlo_fptrunc_fsub_f16_src:
-; GFX1250: ; %bb.0: ; %.entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[0,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-FAKE16-LABEL: mixlo_fptrunc_fsub_f16_src:
+; GFX1250-FAKE16: ; %bb.0: ; %.entry
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_fma_mixlo_bf16 v0, -v1, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-REAL16-LABEL: mixlo_fptrunc_fsub_f16_src:
+; GFX1250-REAL16: ; %bb.0: ; %.entry
+; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-REAL16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-REAL16-NEXT: v_fma_mixlo_bf16 v0, -v1, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
+; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
.entry:
%a.ext = fpext half %a to float
%sub = fsub float %a.ext, %b
@@ -738,14 +750,23 @@ define bfloat @mixlo_fptrunc_fsub_f16_src(half %a, float %b) #0 {
}
define bfloat @mixlo_fptrunc_fsub_f16_src1(float %a, half %b) #0 {
-; GFX1250-LABEL: mixlo_fptrunc_fsub_f16_src1:
-; GFX1250: ; %bb.0: ; %.entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[1,1,0]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-FAKE16-LABEL: mixlo_fptrunc_fsub_f16_src1:
+; GFX1250-FAKE16: ; %bb.0: ; %.entry
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_fma_mixlo_bf16 v0, -v1, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-REAL16-LABEL: mixlo_fptrunc_fsub_f16_src1:
+; GFX1250-REAL16: ; %bb.0: ; %.entry
+; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-REAL16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-REAL16-NEXT: v_fma_mixlo_bf16 v0, -v1, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
+; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
.entry:
%b.ext = fpext half %b to float
%sub = fsub float %a, %b.ext
@@ -758,9 +779,7 @@ define bfloat @mixlo_fptrunc_fsub_abs_src_mod(float %a, float %b) #0 {
; GFX1250: ; %bb.0: ; %.entry
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_sub_f32_e64 v0, v0, |v1|
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_fma_mixlo_bf16 v0, -|v1|, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
.entry:
%b.fabs = call float @llvm.fabs.f32(float %b)
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
index 20e64b9ee74fb..a5e0e737f8bb0 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -2996,34 +2996,22 @@ define half @mixlo_fptrunc_neg_src_mod(float %a, float %b) #0 {
}
define half @mixlo_fptrunc_fadd(float %a, float %b) #0 {
-; SDAG-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fadd:
-; SDAG-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; SDAG-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fadd:
-; SDAG-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: mixlo_fptrunc_fadd:
+; GFX1100: ; %bb.0: ; %.entry
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v1 op_sel_hi:[0,1,0]
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: mixlo_fptrunc_fadd:
; GFX900: ; %bb.0: ; %.entry
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-NEXT: v_mad_mixlo_f16 v0, v0, 1.0, v1 op_sel_hi:[0,1,0]
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: mixlo_fptrunc_fadd:
; GFX906: ; %bb.0: ; %.entry
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX906-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX906-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v1 op_sel_hi:[0,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: mixlo_fptrunc_fadd:
@@ -3040,22 +3028,6 @@ define half @mixlo_fptrunc_fadd(float %a, float %b) #0 {
; SDAG-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fadd:
-; GISEL-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GISEL-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fadd:
-; GISEL-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
; GISEL-CI-LABEL: mixlo_fptrunc_fadd:
; GISEL-CI: ; %bb.0: ; %.entry
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3069,21 +3041,11 @@ define half @mixlo_fptrunc_fadd(float %a, float %b) #0 {
}
define half @mixlo_fptrunc_fadd_no_flush(float %a, float %b) {
-; SDAG-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fadd_no_flush:
-; SDAG-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; SDAG-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fadd_no_flush:
-; SDAG-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: mixlo_fptrunc_fadd_no_flush:
+; GFX1100: ; %bb.0: ; %.entry
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v1 op_sel_hi:[0,1,0]
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: mixlo_fptrunc_fadd_no_flush:
; GFX900: ; %bb.0: ; %.entry
@@ -3095,8 +3057,7 @@ define half @mixlo_fptrunc_fadd_no_flush(float %a, float %b) {
; GFX906-LABEL: mixlo_fptrunc_fadd_no_flush:
; GFX906: ; %bb.0: ; %.entry
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX906-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX906-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v1 op_sel_hi:[0,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: mixlo_fptrunc_fadd_no_flush:
@@ -3113,22 +3074,6 @@ define half @mixlo_fptrunc_fadd_no_flush(float %a, float %b) {
; SDAG-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fadd_no_flush:
-; GISEL-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GISEL-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fadd_no_flush:
-; GISEL-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
; GISEL-CI-LABEL: mixlo_fptrunc_fadd_no_flush:
; GISEL-CI: ; %bb.0: ; %.entry
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3142,21 +3087,11 @@ define half @mixlo_fptrunc_fadd_no_flush(float %a, float %b) {
}
define half @mixlo_fptrunc_fadd_f16_denormals(float %a, float %b) #2 {
-; SDAG-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fadd_f16_denormals:
-; SDAG-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; SDAG-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fadd_f16_denormals:
-; SDAG-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1
-; SDAG-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: mixlo_fptrunc_fadd_f16_denormals:
+; GFX1100: ; %bb.0: ; %.entry
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v1 op_sel_hi:[0,1,0]
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: mixlo_fptrunc_fadd_f16_denormals:
; GFX900: ; %bb.0: ; %.entry
@@ -3168,8 +3103,7 @@ define half @mixlo_fptrunc_fadd_f16_denormals(float %a, float %b) #2 {
; GFX906-LABEL: mixlo_fptrunc_fadd_f16_denormals:
; GFX906: ; %bb.0: ; %.entry
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX906-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX906-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v1 op_sel_hi:[0,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: mixlo_fptrunc_fadd_f16_denormals:
@@ -3186,22 +3120,6 @@ define half @mixlo_fptrunc_fadd_f16_denormals(float %a, float %b) #2 {
; SDAG-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fadd_f16_denormals:
-; GISEL-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GISEL-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fadd_f16_denormals:
-; GISEL-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GISEL-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
; GISEL-CI-LABEL: mixlo_fptrunc_fadd_f16_denormals:
; GISEL-CI: ; %bb.0: ; %.entry
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3215,34 +3133,22 @@ define half @mixlo_fptrunc_fadd_f16_denormals(float %a, float %b) #2 {
}
define half @mixlo_fptrunc_fadd_abs_src_mod(float %a, float %b) #0 {
-; SDAG-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fadd_abs_src_mod:
-; SDAG-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-TRUE16-NEXT: v_add_f32_e64 v0, |v0|, v1
-; SDAG-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; SDAG-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fadd_abs_src_mod:
-; SDAG-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-FAKE16-NEXT: v_add_f32_e64 v0, |v0|, v1
-; SDAG-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: mixlo_fptrunc_fadd_abs_src_mod:
+; GFX1100: ; %bb.0: ; %.entry
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_fma_mixlo_f16 v0, |v0|, 1.0, v1 op_sel_hi:[0,1,0]
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: mixlo_fptrunc_fadd_abs_src_mod:
; GFX900: ; %bb.0: ; %.entry
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_add_f32_e64 v0, |v0|, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-NEXT: v_mad_mixlo_f16 v0, |v0|, 1.0, v1 op_sel_hi:[0,1,0]
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: mixlo_fptrunc_fadd_abs_src_mod:
; GFX906: ; %bb.0: ; %.entry
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_add_f32_e64 v0, |v0|, v1
-; GFX906-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX906-NEXT: v_fma_mixlo_f16 v0, |v0|, 1.0, v1 op_sel_hi:[0,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: mixlo_fptrunc_fadd_abs_src_mod:
@@ -3259,22 +3165,6 @@ define half @mixlo_fptrunc_fadd_abs_src_mod(float %a, float %b) #0 {
; SDAG-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fadd_abs_src_mod:
-; GISEL-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-TRUE16-NEXT: v_add_f32_e64 v0, |v0|, v1
-; GISEL-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GISEL-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fadd_abs_src_mod:
-; GISEL-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-FAKE16-NEXT: v_add_f32_e64 v0, |v0|, v1
-; GISEL-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
; GISEL-CI-LABEL: mixlo_fptrunc_fadd_abs_src_mod:
; GISEL-CI: ; %bb.0: ; %.entry
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3289,34 +3179,22 @@ define half @mixlo_fptrunc_fadd_abs_src_mod(float %a, float %b) #0 {
}
define half @mixlo_fptrunc_fsub(float %a, float %b) #0 {
-; SDAG-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fsub:
-; SDAG-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; SDAG-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; SDAG-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fsub:
-; SDAG-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; SDAG-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: mixlo_fptrunc_fsub:
+; GFX1100: ; %bb.0: ; %.entry
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_fma_mixlo_f16 v0, -v1, 1.0, v0 op_sel_hi:[0,1,0]
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: mixlo_fptrunc_fsub:
; GFX900: ; %bb.0: ; %.entry
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-NEXT: v_mad_mixlo_f16 v0, -v1, 1.0, v0 op_sel_hi:[0,1,0]
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: mixlo_fptrunc_fsub:
; GFX906: ; %bb.0: ; %.entry
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX906-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX906-NEXT: v_fma_mixlo_f16 v0, -v1, 1.0, v0 op_sel_hi:[0,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: mixlo_fptrunc_fsub:
@@ -3333,22 +3211,6 @@ define half @mixlo_fptrunc_fsub(float %a, float %b) #0 {
; SDAG-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fsub:
-; GISEL-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GISEL-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GISEL-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fsub:
-; GISEL-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GISEL-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
; GISEL-CI-LABEL: mixlo_fptrunc_fsub:
; GISEL-CI: ; %bb.0: ; %.entry
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3362,34 +3224,22 @@ define half @mixlo_fptrunc_fsub(float %a, float %b) #0 {
}
define half @mixlo_fptrunc_fsub_abs_src_mod(float %a, float %b) #0 {
-; SDAG-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fsub_abs_src_mod:
-; SDAG-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-TRUE16-NEXT: v_sub_f32_e64 v0, v0, |v1|
-; SDAG-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; SDAG-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fsub_abs_src_mod:
-; SDAG-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-FAKE16-NEXT: v_sub_f32_e64 v0, v0, |v1|
-; SDAG-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: mixlo_fptrunc_fsub_abs_src_mod:
+; GFX1100: ; %bb.0: ; %.entry
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_fma_mixlo_f16 v0, -|v1|, 1.0, v0 op_sel_hi:[0,1,0]
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: mixlo_fptrunc_fsub_abs_src_mod:
; GFX900: ; %bb.0: ; %.entry
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_sub_f32_e64 v0, v0, |v1|
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-NEXT: v_mad_mixlo_f16 v0, -|v1|, 1.0, v0 op_sel_hi:[0,1,0]
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: mixlo_fptrunc_fsub_abs_src_mod:
; GFX906: ; %bb.0: ; %.entry
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_sub_f32_e64 v0, v0, |v1|
-; GFX906-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX906-NEXT: v_fma_mixlo_f16 v0, -|v1|, 1.0, v0 op_sel_hi:[0,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: mixlo_fptrunc_fsub_abs_src_mod:
@@ -3406,22 +3256,6 @@ define half @mixlo_fptrunc_fsub_abs_src_mod(float %a, float %b) #0 {
; SDAG-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fsub_abs_src_mod:
-; GISEL-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-TRUE16-NEXT: v_sub_f32_e64 v0, v0, |v1|
-; GISEL-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GISEL-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fsub_abs_src_mod:
-; GISEL-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-FAKE16-NEXT: v_sub_f32_e64 v0, v0, |v1|
-; GISEL-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
; GISEL-CI-LABEL: mixlo_fptrunc_fsub_abs_src_mod:
; GISEL-CI: ; %bb.0: ; %.entry
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3436,34 +3270,22 @@ define half @mixlo_fptrunc_fsub_abs_src_mod(float %a, float %b) #0 {
}
define half @mixlo_fptrunc_fadd_f16_src(half %a, float %b) #0 {
-; SDAG-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fadd_f16_src:
-; SDAG-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-TRUE16-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; SDAG-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; SDAG-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fadd_f16_src:
-; SDAG-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-FAKE16-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; SDAG-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: mixlo_fptrunc_fadd_f16_src:
+; GFX1100: ; %bb.0: ; %.entry
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: mixlo_fptrunc_fadd_f16_src:
; GFX900: ; %bb.0: ; %.entry
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mad_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-NEXT: v_mad_mixlo_f16 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: mixlo_fptrunc_fadd_f16_src:
; GFX906: ; %bb.0: ; %.entry
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; GFX906-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX906-NEXT: v_fma_mixlo_f16 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: mixlo_fptrunc_fadd_f16_src:
@@ -3482,22 +3304,6 @@ define half @mixlo_fptrunc_fadd_f16_src(half %a, float %b) #0 {
; SDAG-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fadd_f16_src:
-; GISEL-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-TRUE16-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; GISEL-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GISEL-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fadd_f16_src:
-; GISEL-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-FAKE16-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; GISEL-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
; GISEL-CI-LABEL: mixlo_fptrunc_fadd_f16_src:
; GISEL-CI: ; %bb.0: ; %.entry
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3513,34 +3319,22 @@ define half @mixlo_fptrunc_fadd_f16_src(half %a, float %b) #0 {
}
define half @mixlo_fptrunc_fsub_f16_src(half %a, float %b) #0 {
-; SDAG-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fsub_f16_src:
-; SDAG-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-TRUE16-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[0,1,1]
-; SDAG-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; SDAG-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fsub_f16_src:
-; SDAG-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-FAKE16-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[0,1,1]
-; SDAG-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: mixlo_fptrunc_fsub_f16_src:
+; GFX1100: ; %bb.0: ; %.entry
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_fma_mixlo_f16 v0, -v1, 1.0, v0 op_sel_hi:[0,1,1]
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: mixlo_fptrunc_fsub_f16_src:
; GFX900: ; %bb.0: ; %.entry
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mad_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[0,1,1]
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-NEXT: v_mad_mixlo_f16 v0, -v1, 1.0, v0 op_sel_hi:[0,1,1]
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: mixlo_fptrunc_fsub_f16_src:
; GFX906: ; %bb.0: ; %.entry
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[0,1,1]
-; GFX906-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX906-NEXT: v_fma_mixlo_f16 v0, -v1, 1.0, v0 op_sel_hi:[0,1,1]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: mixlo_fptrunc_fsub_f16_src:
@@ -3559,22 +3353,6 @@ define half @mixlo_fptrunc_fsub_f16_src(half %a, float %b) #0 {
; SDAG-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fsub_f16_src:
-; GISEL-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-TRUE16-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[0,1,1]
-; GISEL-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GISEL-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fsub_f16_src:
-; GISEL-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-FAKE16-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[0,1,1]
-; GISEL-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
; GISEL-CI-LABEL: mixlo_fptrunc_fsub_f16_src:
; GISEL-CI: ; %bb.0: ; %.entry
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3590,34 +3368,22 @@ define half @mixlo_fptrunc_fsub_f16_src(half %a, float %b) #0 {
}
define half @mixlo_fptrunc_fsub_f16_src1(float %a, half %b) #0 {
-; SDAG-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fsub_f16_src1:
-; SDAG-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-TRUE16-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[1,1,0]
-; SDAG-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; SDAG-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fsub_f16_src1:
-; SDAG-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; SDAG-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-FAKE16-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[1,1,0]
-; SDAG-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SDAG-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: mixlo_fptrunc_fsub_f16_src1:
+; GFX1100: ; %bb.0: ; %.entry
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_fma_mixlo_f16 v0, -v1, 1.0, v0 op_sel_hi:[1,1,0]
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: mixlo_fptrunc_fsub_f16_src1:
; GFX900: ; %bb.0: ; %.entry
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mad_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[1,1,0]
-; GFX900-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX900-NEXT: v_mad_mixlo_f16 v0, -v1, 1.0, v0 op_sel_hi:[1,1,0]
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: mixlo_fptrunc_fsub_f16_src1:
; GFX906: ; %bb.0: ; %.entry
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[1,1,0]
-; GFX906-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX906-NEXT: v_fma_mixlo_f16 v0, -v1, 1.0, v0 op_sel_hi:[1,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: mixlo_fptrunc_fsub_f16_src1:
@@ -3636,22 +3402,6 @@ define half @mixlo_fptrunc_fsub_f16_src1(float %a, half %b) #0 {
; SDAG-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX1100-TRUE16-LABEL: mixlo_fptrunc_fsub_f16_src1:
-; GISEL-GFX1100-TRUE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-TRUE16-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[1,1,0]
-; GISEL-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GISEL-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX1100-FAKE16-LABEL: mixlo_fptrunc_fsub_f16_src1:
-; GISEL-GFX1100-FAKE16: ; %bb.0: ; %.entry
-; GISEL-GFX1100-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-FAKE16-NEXT: v_fma_mix_f32 v0, -v1, 1.0, v0 op_sel_hi:[1,1,0]
-; GISEL-GFX1100-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX1100-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
; GISEL-CI-LABEL: mixlo_fptrunc_fsub_f16_src1:
; GISEL-CI: ; %bb.0: ; %.entry
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
More information about the llvm-branch-commits
mailing list