[llvm-branch-commits] [llvm] [AMDGPU] Fold fpround of fadd and fsub into v_mad/fma_mixlo and mixhi (PR #224913)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Sep 20 04:16:18 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-globalisel
Author: Dmitry Sidorov (MrSidims)
<details>
<summary>Changes</summary>
MadFmaMixFP32Pats turns (fadd x, y) into (fma x, 1.0, y) and (fsub x, y)
into (fma (-y), 1.0, x) so the mix instructions absorb the operation along
with the f16 or bf16 source modifiers. MadFmaMixFP16Pats and
MadFmaMixFP16Pats_t16 only did this for fmul, so a rounded result still
needed a separate convert for a rounding the mix instructions perform
themselves.
Unlike the f32 patterns these do not require an operand to be an fpextend
of an f16, since an fpround on the result always removes the convert. The
rewrite is exact because the mix instructions round the f32 result again
when they write the 16-bit destination, so it stays f32_to_f16(fma(x, 1.0,
y)).
Assisted-by: Claude Code Opus 5
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
---
Patch is 134.16 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/224913.diff
10 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/VOP3PInstructions.td (+70)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll (+192-285)
- (modified) llvm/test/CodeGen/AMDGPU/bf16.ll (+5-9)
- (modified) llvm/test/CodeGen/AMDGPU/fdiv.f16.ll (+3-8)
- (modified) llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll (+8-16)
- (modified) llvm/test/CodeGen/AMDGPU/frem.ll (+26-52)
- (modified) llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll (+14-23)
- (modified) llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll (+62-145)
- (modified) llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll (+55-36)
- (modified) llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll (+61-311)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index d086b59001b1e..7d73bb328c590 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -300,12 +300,17 @@ multiclass MadFmaMixFP32Pats<SDPatternOperator fma_like,
DSTCLAMP.NONE)>;
}
+// The mix instructions round the f32 result again when they write the f16 or
+// bf16 destination, so these folds match a separate convert exactly.
multiclass MadFmaMixFP16Pats<SDPatternOperator fma_like,
Instruction mixlo_inst,
Instruction mixhi_inst,
ValueType VT = f16,
ValueType vecVT = v2f16> {
defvar VOP3PMadMixModsPat = !if (!eq(VT, bf16), VOP3PMadMixBF16Mods, VOP3PMadMixMods);
+ defvar VOP3PMadMixModsNegPat = !if (!eq(VT, bf16), VOP3PMadMixBF16ModsNeg, VOP3PMadMixModsNeg);
+ defvar OneImm = !if (!eq(VT, bf16), CONST.BF16_ONE, CONST.FP16_ONE);
+ defvar ImmMods = !if (!eq(VT, bf16), !or(SRCMODS.OP_SEL_0, SRCMODS.OP_SEL_1), SRCMODS.OP_SEL_1);
def : GCNPat <
(AMDGPUclamp (build_vector
(VT (fpround (fma_like (f32 (VOP3PMadMixModsPat VT:$lo_src0, i32:$lo_src0_modifiers)),
@@ -345,6 +350,48 @@ multiclass MadFmaMixFP16Pats<SDPatternOperator fma_like,
VGPR_32:$elt0))
>;
+ // (fadd x, y) -> (fma x, 1.0, y)
+ def : GCNPat <
+ (VT (fpround (fadd (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsPat f32:$src1, i32:$src1_modifiers))))),
+ (mixlo_inst $src0_modifiers, $src0,
+ (i32 ImmMods), (i32 OneImm),
+ $src1_modifiers, $src1,
+ DSTCLAMP.NONE,
+ (i32 (IMPLICIT_DEF)))
+ >;
+
+ def : GCNPat <
+ (build_vector VT:$elt0, (VT (fpround (fadd (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsPat f32:$src1, i32:$src1_modifiers)))))),
+ (vecVT (mixhi_inst $src0_modifiers, $src0,
+ (i32 ImmMods), (i32 OneImm),
+ $src1_modifiers, $src1,
+ DSTCLAMP.NONE,
+ VGPR_32:$elt0))
+ >;
+
+ // (fsub x, y) -> (fma (-y), 1.0, x)
+ def : GCNPat <
+ (VT (fpround (fsub (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsNegPat f32:$src1, i32:$src1_modifiers))))),
+ (mixlo_inst $src1_modifiers, $src1,
+ (i32 ImmMods), (i32 OneImm),
+ $src0_modifiers, $src0,
+ DSTCLAMP.NONE,
+ (i32 (IMPLICIT_DEF)))
+ >;
+
+ def : GCNPat <
+ (build_vector VT:$elt0, (VT (fpround (fsub (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsNegPat f32:$src1, i32:$src1_modifiers)))))),
+ (vecVT (mixhi_inst $src1_modifiers, $src1,
+ (i32 ImmMods), (i32 OneImm),
+ $src0_modifiers, $src0,
+ DSTCLAMP.NONE,
+ VGPR_32:$elt0))
+ >;
+
def : GCNPat <
(VT (fpround (fma_like (f32 (VOP3PMadMixModsPat VT:$src0, i32:$src0_modifiers)),
(f32 (VOP3PMadMixModsPat VT:$src1, i32:$src1_modifiers)),
@@ -389,6 +436,9 @@ multiclass MadFmaMixFP16Pats_t16<SDPatternOperator fma_like,
ValueType VT = f16,
ValueType vecVT = v2f16> {
defvar VOP3PMadMixModsPat = !if (!eq(VT, bf16), VOP3PMadMixBF16Mods, VOP3PMadMixMods);
+ defvar VOP3PMadMixModsNegPat = !if (!eq(VT, bf16), VOP3PMadMixBF16ModsNeg, VOP3PMadMixModsNeg);
+ defvar OneImm = !if (!eq(VT, bf16), CONST.BF16_ONE, CONST.FP16_ONE);
+ defvar ImmMods = !if (!eq(VT, bf16), !or(SRCMODS.OP_SEL_0, SRCMODS.OP_SEL_1), SRCMODS.OP_SEL_1);
def : GCNPat <
(VT (fpround (fmul (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
(f32 (VOP3PMadMixModsPat f32:$src1, i32:$src1_modifiers))))),
@@ -398,6 +448,26 @@ multiclass MadFmaMixFP16Pats_t16<SDPatternOperator fma_like,
DSTCLAMP.NONE)
>;
+ // (fadd x, y) -> (fma x, 1.0, y)
+ def : GCNPat <
+ (VT (fpround (fadd (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsPat f32:$src1, i32:$src1_modifiers))))),
+ (mix_inst_16 $src0_modifiers, $src0,
+ (i32 ImmMods), (i32 OneImm),
+ $src1_modifiers, $src1,
+ DSTCLAMP.NONE)
+ >;
+
+ // (fsub x, y) -> (fma (-y), 1.0, x)
+ def : GCNPat <
+ (VT (fpround (fsub (f32 (VOP3PMadMixModsPat f32:$src0, i32:$src0_modifiers)),
+ (f32 (VOP3PMadMixModsNegPat f32:$src1, i32:$src1_modifiers))))),
+ (mix_inst_16 $src1_modifiers, $src1,
+ (i32 ImmMods), (i32 OneImm),
+ $src0_modifiers, $src0,
+ DSTCLAMP.NONE)
+ >;
+
def : GCNPat <
(VT (fpround (fma_like (f32 (VOP3PMadMixModsPat VT:$src0, i32:$src0_modifiers)),
(f32 (VOP3PMadMixModsPat VT:$src1, i32:$src1_modifiers)),
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
index a42e915a4b479..6a76113ac3817 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
@@ -125,8 +125,7 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v1, v3, v0 op_sel_hi:[1,0,1]
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
@@ -144,8 +143,7 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
;
@@ -161,8 +159,7 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v4
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v3
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
@@ -178,8 +175,7 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX11-NEXT: v_fma_mix_f32 v4, -v1, v3, v0 op_sel_hi:[1,0,1]
; GFX11-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX11-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX11-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv half %a, %b
@@ -332,8 +328,7 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v4, -v1, v3, v0 op_sel_hi:[1,0,1]
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
@@ -351,8 +346,7 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
;
@@ -368,8 +362,7 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v4
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v3
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v5 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
@@ -385,8 +378,7 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX11-NEXT: v_fma_mix_f32 v4, -v1, v3, v0 op_sel_hi:[1,0,1]
; GFX11-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX11-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX11-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX11-NEXT: v_fma_mixlo_f16 v2, v2, 1.0, v3 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv half %a, %b
@@ -942,28 +934,26 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v3
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v0, v2, neg(0) op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v2
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v8, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v0, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v6, v0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v6, v7, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v6, v0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v7, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v7, v6, v4
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v4, v5, v4
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v4, v4, v7
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v3, v5
-; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v8, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v8, v6, v4
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v2, v2, v1, v0
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v0, -v1, v8, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, v0, 1.0, v8 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v3, v5
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-IEEE-LABEL: v_fdiv_v2f16:
@@ -989,16 +979,14 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, v0, 1.0, v7 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, v0, 1.0, v8 op_sel:[1,0,0] op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v7, 16, v0
; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v8, v4
; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX10-IEEE-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX10-IEEE-NEXT: v_add_f32_e32 v4, v4, v6
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v4, v4, 1.0, v6 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v3, v1, v0
-; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v4, v2, v5
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v4, v2, v7
; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
;
@@ -1025,10 +1013,8 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v4, v4, v6
; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX10-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v9
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v4, v4, v11
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v9 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v4, v4, 1.0, v11 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v1, v0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v2, v7
; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -1050,15 +1036,14 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX11-NEXT: v_dual_fmac_f32 v5, v7, v3 :: v_dual_fmac_f32 v6, v8, v4
; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel_hi:[1,0,1]
; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_dual_mul_f32 v3, v7, v3 :: v_dual_mul_f32 v4, v8, v4
+; GFX11-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-NEXT: v_dual_mul_f32 v4, v8, v4 :: v_dual_and_b32 v3, 0xff800000, v3
; GFX11-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX11-NEXT: v_dual_add_f32 v4, v4, v6 :: v_dual_and_b32 v3, 0xff800000, v3
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX11-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX11-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX11-NEXT: v_fma_mixlo_f16 v4, v4, 1.0, v6 op_sel_hi:[0,1,0]
; GFX11-NEXT: v_div_fixup_f16 v0, v3, v1, v0
-; GFX11-NEXT: v_div_fixup_f16 v1, v4, v2, v5
+; GFX11-NEXT: v_div_fixup_f16 v1, v4, v2, v7
; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fdiv = fdiv <2 x half> %a, %b
@@ -1337,28 +1322,26 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v3
+; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, v0, v2, neg(0) op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v2
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, -v1, v5, v0 op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v8, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, v0, v2, neg(0) op_sel_hi:[1,0,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v6, v0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v6, v7, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v7, -v1, v6, v0 op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v2, v7, v2
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v8, v0, v4, neg(0) op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v7, v6, v4
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX9-FLUSH-NEXT: v_mad_mix_f32 v5, -v1, v7, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX9-FLUSH-NEXT: v_mul_f32_e32 v4, v5, v4
-; GFX9-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX9-FLUSH-NEXT: v_add_f32_e32 v4, v4, v7
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX9-FLUSH-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
-; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v3, v5
-; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, v2, 1.0, v6 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v6, -v1, v8, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mac_f32_e32 v8, v6, v4
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v2, v2, v1, v0
+; GFX9-FLUSH-NEXT: v_mad_mix_f32 v0, -v1, v8, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
+; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v0, v0, 1.0, v8 op_sel_hi:[0,1,0]
+; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v3, v5
+; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-IEEE-LABEL: v_fdiv_v2f16_ulp25:
@@ -1384,16 +1367,14 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX10-IEEE-NEXT: v_fma_mix_f32 v7, v0, 1.0, v7 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v8, v0, 1.0, v8 op_sel:[1,0,0] op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v7, 16, v0
; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v8, v4
; GFX10-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX10-IEEE-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX10-IEEE-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX10-IEEE-NEXT: v_add_f32_e32 v4, v4, v6
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v5 op_sel_hi:[0,1,0]
+; GFX10-IEEE-NEXT: v_fma_mixlo_f16 v4, v4, 1.0, v6 op_sel_hi:[0,1,0]
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v3, v1, v0
-; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v4, v2, v5
+; GFX10-IEEE-NEXT: v_div_fixup_f16 v1, v4, v2, v7
; GFX10-IEEE-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
;
@@ -1420,10 +1401,8 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v4, v4, v6
; GFX10-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
; GFX10-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v3, v3, v9
-; GFX10-FLUSH-NEXT: v_add_f32_e32 v4, v4, v11
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v3, v3, 1.0, v9 op_sel_hi:[0,1,0]
+; GFX10-FLUSH-NEXT: v_fma_mixlo_f16 v4, v4, 1.0, v11 op_sel_hi:[0,1,0]
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v1, v0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v2, v7
; GFX10-FLUSH-NEXT: v_pack_b32_f16 v0, v0, v1
@@ -1445,15 +1424,14 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX11-NEXT: v_dual_fmac_f32 v5, v7, v3 :: v_dual_fmac_f32 v6, v8, v4
; GFX11-NEXT: v_fma_mix_f32 v7, -v1, v5, v0 op_sel_hi:[1,0,1]
; GFX11-NEXT: v_fma_mix_f32 v8, -v1, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX11-NEXT: v_dual_mul_f32 v3, v7, v3 :: v_dual_mul_f32 v4, v8, v4
+; GFX11-NEXT: v_mul_f32_e32 v3, v7, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/224913
More information about the llvm-branch-commits
mailing list